CUNY 2020 Analysis
Packages and utilities
library(tidyverse)
library(lme4)
library(lmerTest)
library(logging)
library(mvtnorm)
library(mgcv)# Compute the log-likelihood of a new dataset using a fit lme4 model.
logLik_test <- function(lm, test_X, test_y) {
predictions <- predict(lm, test_X, re.form=NA)
# Get std.dev. of residual, estimated from train data
stdev <- sigma(lm)
# For each prediction--observation, get the density p(obs | N(predicted, model_sigma)) and reduce
density <- sum(dnorm(test_y, predictions, stdev, log=TRUE))
return(density)
}
# Get per-prediction log-likelihood
logLik_test_per <- function(lm, test_X, test_y) {
predictions <- predict(lm, test_X, re.form=NA)
# Get std.dev. of residual, estimated from train data
stdev <- sigma(lm)
# For each prediction--observation, get the density p(obs | N(predicted, model_sigma))
densities <- dnorm(test_y, predictions, stdev, log=TRUE)
return(densities)
}
# Compute MSE of a new dataset using a fit lme4 model.
mse_test <- function(lm, test_X, test_y) {
return(mean((predict(lm, test_X, re.form=NA) - test_y) ^ 2))
}
#Sanity checks
#mylm <- gam(psychometric ~ s(surprisal, bs = "cr", k = 20) + s(prev_surp, bs = "cr", k = 20) + te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr"), data=train_data)
#c(logLik(mylm), logLik_test(mylm, train_data, train_data$psychometric))
#logLik_test(mylm, test_data, test_data$psychometric)Data loading and preprocessing
data = read.csv("../data/harmonized_results.csv")
all_data = data %>%
mutate(seed = as.factor(seed)) %>%
group_by(corpus, model, training, seed) %>%
mutate(prev_surp = lag(surprisal),
prev_code = lag(code),
prev_len = lag(len),
prev_freq = lag(freq),
prev_surp = lag(surprisal),
prev2_freq = lag(prev_freq),
prev2_code = lag(prev_code),
prev2_len = lag(prev_len),
prev2_surp = lag(prev_surp),
prev3_freq = lag(prev2_freq),
prev3_code = lag(prev2_code),
prev3_len = lag(prev2_len),
prev3_surp = lag(prev2_surp)) %>%
ungroup() %>%
# Filter back three for the dundee corpus. Filter back 1 for all other corpora
# NB this effectively removes all zero-surprisal rows, since early-sentence tokens don't have contiguous token history
filter((corpus == "dundee" & code == prev2_code + 2) | (corpus != "dundee" & code == prev_code + 1)) %>%
select(-prev_code, -prev2_code, -prev3_code) %>%
drop_na()
all_data = all_data %>%
mutate(
model = as.character(model),
model = if_else(model == "gpt-2", "gpt2", model),
model = as.factor(model))missing_rows = all_data %>% complete(nesting(corpus, code), nesting(model, training, seed)) %>%
group_by(corpus, code) %>%
filter(sum(is.na(surprisal)) > 0) %>%
ungroup() %>%
anti_join(all_data, by=c("corpus", "code", "model", "training", "seed"))
missing_rows %>% ggplot(aes(x=corpus, fill=factor(paste(model,training)))) + geom_bar(position=position_dodge(width=0.8))
print(missing_rows %>% group_by(model, training, seed, corpus) %>% summarise(n=n())) %>% arrange(desc(n))
# Compute the ideal number of model--seed--training observations per token.
to_drop = all_data %>%
group_by(corpus, code) %>% summarise(n = n()) %>% ungroup() %>%
group_by(corpus) %>% mutate( max_n = max(n)) %>% ungroup() %>%
filter(max_n != n) %>%
select(code, corpus)
#to_drop = all_data %>% group_by(corpus, code) %>% filter(n() != ideal_token_obs_count) %>% ungroup()
loginfo(paste("Dropping", nrow(to_drop), "observations corresponding to corpus tokens which are missing observations for some model."))[0m2020-05-08 13:56:21 INFO::Dropping 9628 observations corresponding to corpus tokens which are missing observations for some model.[0m[0m[0mloginfo(paste("Dropping", to_drop %>% group_by(corpus, code) %>% n_groups(), "tokens which are missing observations for some model."))[0m2020-05-08 13:56:21 INFO::Dropping 9628 tokens which are missing observations for some model.[0m[0m[0mall_data = all_data %>% anti_join(to_drop %>% group_by(corpus, code), by=c("corpus", "code"))
loginfo(paste("After drop,", nrow(all_data), "observations (", all_data %>% group_by(corpus, code) %>% n_groups(), " tokens) remain."))[0m2020-05-08 13:56:21 INFO::After drop, 1128235 observations ( 38757 tokens) remain.[0m[0m[0m
to_drop_zero_surps = all_data %>% group_by(corpus, code) %>% filter(any(surprisal == 0)) %>% ungroup()
loginfo(paste("Dropping", nrow(to_drop_zero_surps), "observations corresponding to corpus tokens which have surprisal zeros for some model."))[0m2020-05-08 13:56:22 INFO::Dropping 146 observations corresponding to corpus tokens which have surprisal zeros for some model.[0m[0m[0mloginfo(paste("Dropping", to_drop_zero_surps %>% group_by(corpus, code) %>% n_groups(), "tokens which have surprisal zeros for some model."))[0m2020-05-08 13:56:22 INFO::Dropping 5 tokens which have surprisal zeros for some model.[0m[0m[0mall_data = all_data %>% anti_join(to_drop_zero_surps %>% group_by(corpus, code), by=c("corpus", "code"))
loginfo(paste("After drop,", nrow(all_data), "observations (", all_data %>% group_by(corpus, code) %>% n_groups(), " tokens) remain."))[0m2020-05-08 13:56:22 INFO::After drop, 1128089 observations ( 38752 tokens) remain.[0m[0m[0m
to_drop_zero_psychs = all_data %>% group_by(corpus, code) %>% filter(any(psychometric == 0)) %>% ungroup()
loginfo(paste("Dropping", nrow(to_drop_zero_psychs), "observations corresponding to corpus tokens which have surprisal zeros for some model."))[0m2020-05-08 13:56:23 INFO::Dropping 14935 observations corresponding to corpus tokens which have surprisal zeros for some model.[0m[0m[0mloginfo(paste("Dropping", to_drop_zero_psychs %>% group_by(corpus, code) %>% n_groups(), "tokens which have surprisal zeros for some model."))[0m2020-05-08 13:56:23 INFO::Dropping 515 tokens which have surprisal zeros for some model.[0m[0m[0mall_data = all_data %>% anti_join(to_drop_zero_psychs %>% group_by(corpus, code), by=c("corpus", "code"))
loginfo(paste("After drop,", nrow(all_data), "observations (", all_data %>% group_by(corpus, code) %>% n_groups(), " tokens) remain."))[0m2020-05-08 13:56:23 INFO::After drop, 1113154 observations ( 38237 tokens) remain.[0m[0m[0mLearn models
# Compute linear model stats for the given training data subset and full test data.
# Automatically subsets the test data to match the relevant group for which we are training a linear model.
get_lm_data <- function(df, test_data, formula, fold, store_env) {
#this_lm <- gam(formula, data=df);
this_lm = lm(formula, data=df)
this_test_data <- semi_join(test_data, df, by=c("training", "model", "seed", "corpus"));
# Save lm to the global env so that we can access residuals later.
lm_name = paste(unique(paste(df$model, df$training, df$seed, df$corpus))[1], fold)
assign(lm_name, this_lm, envir=store_env)
summarise(df,
log_lik = as.numeric(logLik(this_lm, REML = F)),
test_lik = logLik_test(this_lm, this_test_data, this_test_data$psychometric),
test_mse = mse_test(this_lm, this_test_data, this_test_data$psychometric))
}
# For a previously fitted lm stored in store_env, get the residuals on test data of the relevant data subset.
get_lm_residuals <- function(df, fold, store_env) {
# Retrieve the relevant lm.
lm_name = paste(unique(paste(df$model, df$training, df$seed, df$corpus))[1], fold)
this_lm <- get(lm_name, envir=store_env)
mutate(df,
likelihood = logLik_test_per(this_lm, df, df$psychometric),
resid = df$psychometric - predict(this_lm, df, re.form=NA))
}
# Compute per-example delta-log-likelihood for the given test fold.
get_lm_delta_log_lik <- function(test_data, fold, baseline_env, full_env) {
lm_name = paste(unique(paste(test_data$model, test_data$training, test_data$seed, test_data$corpus))[1], fold)
baseline_lm <- get(lm_name, envir=baseline_env)
full_lm <- get(lm_name, envir=full_env)
delta_log_lik = logLik_test_per(full_lm, test_data, test_data$psychometric) - logLik_test_per(baseline_lm, test_data, test_data$psychometric)
return(cbind(test_data, delta_log_lik=delta_log_lik))
}
#####
# Define regression formulae.
# Eye-tracking regression: only use surprisal and previous surprisal; SPRT regression: use 2-back features.
#baseline_rt_regression = psychometric ~ te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr")
#baselie_sprt_regression = psychometric ~ te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr") + te(prev2_freq, prev2_len, bs = "cr")
#full_rt_regression = (psychometric ~ s(surprisal, bs = "cr", k = 20) + s(prev_surp, bs = "cr", k = 20)
#+ te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr"))
#full_sprt_regression = (psychometric ~ s(surprisal, bs = "cr", k = 20) + s(prev_surp, bs = "cr", k = 20) + s(prev2_surp, bs = "cr", k = 20)
#+ te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr") + te(prev2_freq, prev2_len, bs = "cr"))
baseline_rt_regression = psychometric ~ freq + prev_freq + prev2_freq + prev3_freq + len + prev_len + prev2_len + prev3_len
baseline_sprt_regression = psychometric ~ freq + prev_freq + len + prev_len
full_rt_regression = psychometric ~ surprisal + prev_surp + prev2_surp + prev3_surp + freq + prev_freq + prev2_freq + prev3_freq + len + prev_len + prev2_len + prev3_len
full_sprt_regression = psychometric ~ surprisal + prev_surp + freq + prev_freq + len + prev_len
#####
# Prepare frames/environments for storing results/objects.
baseline_results = data.frame()
full_model_results = data.frame()
baseline_residuals = data.frame()
full_residuals = data.frame()
log_lik_deltas = data.frame()
#Randomly shuffle the data
all_data<-all_data[sample(nrow(all_data)),]
#Create K equally size folds
K = 5
folds <- cut(seq(1,nrow(all_data)),breaks=K,labels=FALSE)
#Perform 10 fold cross validation
# Fit models for some fold of the data.
baseline_corpus = function(corpus, df, test_data, fold, env) {
if(corpus == "dundee") {
get_lm_data(df, test_data, baseline_rt_regression, fold, env)
} else {
get_lm_data(df, test_data, baseline_sprt_regression, fold, env)
}
}
full_model_corpus = function(corpus, df, test_data, fold, env) {
if(corpus[1] == "dundee") {
get_lm_data(df, test_data, full_rt_regression, fold, env)
} else {
get_lm_data(df, test_data, full_sprt_regression, fold, env)
}
}
# Prepare a new Environment in which we store fitted LMs, which we'll query later for residuals and other metrics.
baseline_env = new.env()
full_env = new.env()
for(i in 1:K) {
#Segement your data by fold using the which() function
testIndexes <- which(folds==i, arr.ind=TRUE)
test_data <- all_data[testIndexes, ]
train_data <- all_data[-testIndexes, ]
# Compute a baseline linear model for each model--training--seed--RT-corpus combination.
baselines = train_data %>%
group_by(model, training, seed, corpus) %>%
print(model) %>%
do(baseline_corpus(unique(.$corpus), ., test_data, i, baseline_env)) %>%
ungroup() %>%
mutate(seed = as.factor(seed),
fold = i)
baseline_results = rbind(baseline_results, baselines)
# Compute a full linear model for each model--training--seed-RT-corpus combination
full_models = train_data %>%
group_by(model, training, seed, corpus) %>%
do(full_model_corpus(unique(.$corpus), ., test_data, i, full_env)) %>%
ungroup() %>%
mutate(seed = as.factor(seed),
fold = i)
full_model_results = rbind(full_model_results, full_models)
# Compute delta-log-likelihoods
fold_log_lik_deltas = test_data %>%
group_by(model, training, seed, corpus) %>%
do(get_lm_delta_log_lik(., i, baseline_env, full_env)) %>%
ungroup()
log_lik_deltas = rbind(log_lik_deltas, fold_log_lik_deltas)
fold_baseline_residuals = test_data %>%
group_by(model, training, seed, corpus) %>%
do(get_lm_residuals(., i, baseline_env)) %>%
ungroup()
baseline_residuals = rbind(baseline_residuals, fold_baseline_residuals)
fold_full_residuals = test_data %>%
group_by(model, training, seed, corpus) %>%
do(get_lm_residuals(., i, full_env)) %>%
ungroup()
full_residuals = rbind(full_residuals, fold_full_residuals)
}
|============================================================================================================== | 61% ~1 s remaining
|================================================================================================================== | 64% ~1 s remaining
|====================================================================================================================== | 66% ~1 s remaining
|======================================================================================================================== | 67% ~1 s remaining
|============================================================================================================================ | 69% ~1 s remaining
|============================================================================================================================== | 70% ~1 s remaining
|================================================================================================================================== | 73% ~1 s remaining
|==================================================================================================================================== | 74% ~1 s remaining
|========================================================================================================================================= | 76% ~1 s remaining
|============================================================================================================================================= | 78% ~1 s remaining
|================================================================================================================================================= | 81% ~1 s remaining
|=================================================================================================================================================== | 82% ~1 s remaining
|======================================================================================================================================================= | 84% ~1 s remaining
|========================================================================================================================================================= | 85% ~0 s remaining
|============================================================================================================================================================= | 88% ~0 s remaining
|=============================================================================================================================================================== | 89% ~0 s remaining
|=================================================================================================================================================================== | 91% ~0 s remaining
|===================================================================================================================================================================== | 92% ~0 s remaining
|========================================================================================================================================================================= | 94% ~0 s remaining
|=========================================================================================================================================================================== | 95% ~0 s remaining
|=============================================================================================================================================================================== | 98% ~0 s remaining
|================================================================================================================================================================================= | 99% ~0 s remaining
|============================================================================================================ | 60% ~1 s remaining
|================================================================================================================ | 62% ~1 s remaining
|================================================================================================================== | 64% ~1 s remaining
|====================================================================================================================== | 66% ~1 s remaining
|======================================================================================================================== | 67% ~1 s remaining
|============================================================================================================================ | 69% ~1 s remaining
|============================================================================================================================== | 70% ~1 s remaining
|================================================================================================================================== | 73% ~1 s remaining
|==================================================================================================================================== | 74% ~1 s remaining
|========================================================================================================================================= | 76% ~1 s remaining
|============================================================================================================================================= | 78% ~1 s remaining
|================================================================================================================================================= | 81% ~1 s remaining
|=================================================================================================================================================== | 82% ~1 s remaining
|======================================================================================================================================================= | 84% ~1 s remaining
|========================================================================================================================================================= | 85% ~0 s remaining
|============================================================================================================================================================= | 88% ~0 s remaining
|=============================================================================================================================================================== | 89% ~0 s remaining
|=================================================================================================================================================================== | 91% ~0 s remaining
|===================================================================================================================================================================== | 92% ~0 s remaining
|========================================================================================================================================================================= | 94% ~0 s remaining
|=========================================================================================================================================================================== | 95% ~0 s remaining
|=============================================================================================================================================================================== | 98% ~0 s remaining
|================================================================================================================================================================================= | 99% ~0 s remaining
|================================================================================================================== | 64% ~1 s remaining
|==================================================================================================================== | 65% ~1 s remaining
|======================================================================================================================== | 67% ~1 s remaining
|============================================================================================================================ | 69% ~1 s remaining
|============================================================================================================================== | 70% ~1 s remaining
|================================================================================================================================== | 73% ~1 s remaining
|==================================================================================================================================== | 74% ~1 s remaining
|========================================================================================================================================= | 76% ~1 s remaining
|============================================================================================================================================= | 78% ~1 s remaining
|================================================================================================================================================= | 81% ~1 s remaining
|=================================================================================================================================================== | 82% ~1 s remaining
|======================================================================================================================================================= | 84% ~1 s remaining
|========================================================================================================================================================= | 85% ~0 s remaining
|============================================================================================================================================================= | 88% ~0 s remaining
|=============================================================================================================================================================== | 89% ~0 s remaining
|=================================================================================================================================================================== | 91% ~0 s remaining
|===================================================================================================================================================================== | 92% ~0 s remaining
|========================================================================================================================================================================= | 94% ~0 s remaining
|=========================================================================================================================================================================== | 95% ~0 s remaining
|=============================================================================================================================================================================== | 98% ~0 s remaining
|================================================================================================================================================================================= | 99% ~0 s remaining
|============================================================================================================== | 61% ~1 s remaining
|================================================================================================================== | 64% ~1 s remaining
|====================================================================================================================== | 66% ~1 s remaining
|======================================================================================================================== | 67% ~1 s remaining
|============================================================================================================================ | 69% ~1 s remaining
|============================================================================================================================== | 70% ~1 s remaining
|================================================================================================================================== | 73% ~1 s remaining
|==================================================================================================================================== | 74% ~1 s remaining
|========================================================================================================================================= | 76% ~1 s remaining
|============================================================================================================================================= | 78% ~1 s remaining
|================================================================================================================================================= | 81% ~1 s remaining
|=================================================================================================================================================== | 82% ~1 s remaining
|======================================================================================================================================================= | 84% ~1 s remaining
|========================================================================================================================================================= | 85% ~0 s remaining
|============================================================================================================================================================= | 88% ~0 s remaining
|=============================================================================================================================================================== | 89% ~0 s remaining
|=================================================================================================================================================================== | 91% ~0 s remaining
|===================================================================================================================================================================== | 92% ~0 s remaining
|========================================================================================================================================================================= | 94% ~0 s remaining
|=========================================================================================================================================================================== | 95% ~0 s remaining
|=============================================================================================================================================================================== | 98% ~0 s remaining
|================================================================================================================================================================================= | 99% ~0 s remaining
|================================================================================================================== | 64% ~1 s remaining
|====================================================================================================================== | 66% ~1 s remaining
|======================================================================================================================== | 67% ~1 s remaining
|============================================================================================================================ | 69% ~1 s remaining
|============================================================================================================================== | 70% ~1 s remaining
|================================================================================================================================== | 73% ~1 s remaining
|==================================================================================================================================== | 74% ~1 s remaining
|========================================================================================================================================= | 76% ~1 s remaining
|=========================================================================================================================================== | 77% ~1 s remaining
|============================================================================================================================================= | 78% ~1 s remaining
|================================================================================================================================================= | 81% ~1 s remaining
|=================================================================================================================================================== | 82% ~1 s remaining
|===================================================================================================================================================== | 83% ~1 s remaining
|========================================================================================================================================================= | 85% ~1 s remaining
|=========================================================================================================================================================== | 86% ~0 s remaining
|=============================================================================================================================================================== | 89% ~0 s remaining
|================================================================================================================================================================= | 90% ~0 s remaining
|===================================================================================================================================================================== | 92% ~0 s remaining
|========================================================================================================================================================================= | 94% ~0 s remaining
|=========================================================================================================================================================================== | 95% ~0 s remaining
|=============================================================================================================================================================================== | 98% ~0 s remaining
|================================================================================================================================================================================= | 99% ~0 s remaining
|============================================================================================================ | 60% ~1 s remaining
|================================================================================================================ | 62% ~1 s remaining
|================================================================================================================== | 64% ~1 s remaining
|====================================================================================================================== | 66% ~1 s remaining
|======================================================================================================================== | 67% ~1 s remaining
|============================================================================================================================ | 69% ~1 s remaining
|============================================================================================================================== | 70% ~1 s remaining
|================================================================================================================================== | 73% ~1 s remaining
|==================================================================================================================================== | 74% ~1 s remaining
|========================================================================================================================================= | 76% ~1 s remaining
|============================================================================================================================================= | 78% ~1 s remaining
|================================================================================================================================================= | 81% ~1 s remaining
|=================================================================================================================================================== | 82% ~1 s remaining
|======================================================================================================================================================= | 84% ~1 s remaining
|========================================================================================================================================================= | 85% ~0 s remaining
|============================================================================================================================================================= | 88% ~0 s remaining
|=============================================================================================================================================================== | 89% ~0 s remaining
|=================================================================================================================================================================== | 91% ~0 s remaining
|===================================================================================================================================================================== | 92% ~0 s remaining
|========================================================================================================================================================================= | 94% ~0 s remaining
|=========================================================================================================================================================================== | 95% ~0 s remaining
|=============================================================================================================================================================================== | 98% ~0 s remaining
|================================================================================================================================================================================= | 99% ~0 s remaining
|======================================================================================================================== | 67% ~1 s remaining
|============================================================================================================================ | 69% ~1 s remaining
|============================================================================================================================== | 70% ~1 s remaining
|================================================================================================================================== | 73% ~1 s remaining
|==================================================================================================================================== | 74% ~1 s remaining
|========================================================================================================================================= | 76% ~1 s remaining
|============================================================================================================================================= | 78% ~1 s remaining
|================================================================================================================================================= | 81% ~1 s remaining
|=================================================================================================================================================== | 82% ~1 s remaining
|======================================================================================================================================================= | 84% ~0 s remaining
|========================================================================================================================================================= | 85% ~0 s remaining
|============================================================================================================================================================= | 88% ~0 s remaining
|=============================================================================================================================================================== | 89% ~0 s remaining
|=================================================================================================================================================================== | 91% ~0 s remaining
|===================================================================================================================================================================== | 92% ~0 s remaining
|========================================================================================================================================================================= | 94% ~0 s remaining
|=========================================================================================================================================================================== | 95% ~0 s remaining
|=============================================================================================================================================================================== | 98% ~0 s remaining
|================================================================================================================================================================================= | 99% ~0 s remaining
|==================================================================================================================== | 65% ~1 s remaining
|======================================================================================================================== | 67% ~1 s remaining
|============================================================================================================================ | 69% ~1 s remaining
|============================================================================================================================== | 70% ~1 s remaining
|================================================================================================================================== | 73% ~1 s remaining
|==================================================================================================================================== | 74% ~1 s remaining
|========================================================================================================================================= | 76% ~1 s remaining
|============================================================================================================================================= | 78% ~1 s remaining
|================================================================================================================================================= | 81% ~1 s remaining
|=================================================================================================================================================== | 82% ~1 s remaining
|======================================================================================================================================================= | 84% ~1 s remaining
|========================================================================================================================================================= | 85% ~0 s remaining
|============================================================================================================================================================= | 88% ~0 s remaining
|=============================================================================================================================================================== | 89% ~0 s remaining
|=================================================================================================================================================================== | 91% ~0 s remaining
|===================================================================================================================================================================== | 92% ~0 s remaining
|========================================================================================================================================================================= | 94% ~0 s remaining
|=========================================================================================================================================================================== | 95% ~0 s remaining
|=============================================================================================================================================================================== | 98% ~0 s remaining
|================================================================================================================================================================================= | 99% ~0 s remaining
|============================================================================================================== | 61% ~1 s remaining
|================================================================================================================== | 64% ~1 s remaining
|====================================================================================================================== | 66% ~1 s remaining
|======================================================================================================================== | 67% ~1 s remaining
|============================================================================================================================ | 69% ~1 s remaining
|============================================================================================================================== | 70% ~1 s remaining
|================================================================================================================================== | 73% ~1 s remaining
|==================================================================================================================================== | 74% ~1 s remaining
|========================================================================================================================================= | 76% ~1 s remaining
|============================================================================================================================================= | 78% ~1 s remaining
|================================================================================================================================================= | 81% ~1 s remaining
|=================================================================================================================================================== | 82% ~1 s remaining
|======================================================================================================================================================= | 84% ~1 s remaining
|========================================================================================================================================================= | 85% ~0 s remaining
|============================================================================================================================================================= | 88% ~0 s remaining
|=============================================================================================================================================================== | 89% ~0 s remaining
|=================================================================================================================================================================== | 91% ~0 s remaining
|===================================================================================================================================================================== | 92% ~0 s remaining
|========================================================================================================================================================================= | 94% ~0 s remaining
|=========================================================================================================================================================================== | 95% ~0 s remaining
|=============================================================================================================================================================================== | 98% ~0 s remaining
|================================================================================================================================================================================= | 99% ~0 s remaining
|============================================================================================================== | 61% ~1 s remaining
|================================================================================================================== | 64% ~1 s remaining
|====================================================================================================================== | 66% ~1 s remaining
|======================================================================================================================== | 67% ~1 s remaining
|============================================================================================================================ | 69% ~1 s remaining
|============================================================================================================================== | 70% ~1 s remaining
|================================================================================================================================== | 73% ~1 s remaining
|==================================================================================================================================== | 74% ~1 s remaining
|========================================================================================================================================= | 76% ~1 s remaining
|============================================================================================================================================= | 78% ~1 s remaining
|================================================================================================================================================= | 81% ~1 s remaining
|=================================================================================================================================================== | 82% ~1 s remaining
|======================================================================================================================================================= | 84% ~1 s remaining
|========================================================================================================================================================= | 85% ~0 s remaining
|============================================================================================================================================================= | 88% ~0 s remaining
|=============================================================================================================================================================== | 89% ~0 s remaining
|=================================================================================================================================================================== | 91% ~0 s remaining
|===================================================================================================================================================================== | 92% ~0 s remaining
|========================================================================================================================================================================= | 94% ~0 s remaining
|=========================================================================================================================================================================== | 95% ~0 s remaining
|=============================================================================================================================================================================== | 98% ~0 s remaining
|================================================================================================================================================================================= | 99% ~0 s remaining #write.csv(full_residuals, "../data/analysis_checkpoints/full_residuals.csv")
#write.csv(baseline_residuals, "../data/analysis_checkpoints/baseline_residuals.csv")model_deltas = log_lik_deltas %>%
group_by(model, training, seed, corpus) %>%
summarise(mean_delta_log_lik = mean(delta_log_lik),
sem_delta_log_lik = sd(delta_log_lik) / sqrt(length(delta_log_lik)))write.csv(full_model_results, "../data/analysis_checkpoints/full_model_result.csv")
write.csv(baseline_results, "../data/analysis_checkpoints/baseline_results.csv")
#full_model_results = read.csv("../data/analysis_checkpoints/ffull_model_results.csv")
#baseline_results = read.csv("../data/analysis_checkpoints/fbaseline_resultsb.csv")metric <- "ΔLogLik"
#metric <- "-ΔMSE"
# # Select the relevant metric.
model_deltas = model_deltas %>%
# Retrieve the current test metric
mutate(delta_test_mean = mean_delta_log_lik,
delta_test_sem = sem_delta_log_lik) %>%
# mutate(delta_test_mean = mean_delta_mse,
# delta_test_sem = sem_delta_mse)
# Remove the raw metrics.
select(-mean_delta_log_lik, -sem_delta_log_lik,
#-mean_delta_mse, -sem_delta_mse
)
model_deltas# Sanity check: training on train+test data should yield improved performance over training on just training data. (When evaluating on test data.)
# full_baselines = all_data %>%
# group_by(model, training, seed, corpus) %>%
# summarise(baseline_train_all_test_lik = logLik_test(lm(psychometric ~ len + freq + sent_pos, data=.), semi_join(test_data, ., by=c("training", "model", "seed", "corpus")), semi_join(test_data, ., by=c("training", "model", "seed", "corpus"))$psychometric)) %>%
# ungroup()
# full_baselines
#
# full_baselines %>%
# right_join(baselines, by=c("seed", "training", "model", "corpus")) %>%
# mutate(delta=baseline_train_all_test_lik-baseline_test_lik) %>%
# select(-baseline_lik) # %>%
# #select(-baseline_test_lik, -baseline_train_all_test_lik, -baseline_lik, -baseline_test_mse)Load language model data (SyntaxGym, PPL)
language_model_data = read.csv("../data/model_metadata.csv") %>%
mutate(model = as.character(model),
model = if_else(model == "gpt-2", "gpt2", model),
model = as.factor(model)) %>%
mutate(train_size = case_when(str_starts(training, "bllip-lg") ~ 42,
str_starts(training, "bllip-md") ~ 15,
str_starts(training, "bllip-sm") ~ 5,
str_starts(training, "bllip-xs") ~ 1),
# Training vocabulary usually covaries with the training corpus.
# But BPE models share a vocabulary across training corpora.
training_vocab=as.factor(ifelse(str_detect(training, "gptbpe"), "gptbpe", as.character(training))),
training_source=as.factor(str_replace(as.character(training), "-gptbpe", ""))
) %>%
mutate(seed = as.factor(seed)) %>%
select(-pid, -test_loss) %>%
distinct(model, training, seed, .keep_all = TRUE)
table(language_model_data$seed)
0 111 120 922 1111 3602 4301 7245 7877 28066 28068 44862 51272 64924 1581807512 1581807578 1581861474 1581955288
4 7 6 5 4 1 1 1 1 1 1 1 1 1 1 1 1 1
1582126320 1586986276 1587139950
1 1 1 table(model_deltas$seed)
111 120 607 922 1111 3602 4301 7245 7877 28066 28068 44862 51272 64924 1581807512 1581807578 1581861474 1581955288
9 9 1 9 12 3 3 3 3 3 3 3 3 3 3 3 3 3
1582126320 1586986276 1587139950
3 3 3 First join delta-metric data with model auxiliary data.
model_deltas = model_deltas %>%
merge(language_model_data, by = c("seed", "training", "model"), all=T) %>%
drop_na()
model_deltasAlso join on the original linear model data, rather than collapsing to delta-metrics. This will support regressions later on that don’t collapse across folds.
Final data preprocessing
# Exclude ordered-neurons from all analyses.
model_deltas <- model_deltas %>%
filter(model != "ordered-neurons")Visualizations
The basics
all_data %>% ggplot(aes(x=corpus)) + geom_bar()
print(all_data %>% group_by(corpus) %>% summarise(n=n()))all_data %>%
ggplot(aes(x=freq, color=corpus)) + geom_density()
all_data %>%
ggplot(aes(x=len, color=corpus)) + geom_density()
all_data %>%
ggplot(aes(x=surprisal, color=corpus)) + geom_density()
Predictive power and SG
model_deltas %>%
ggplot(aes(x=sg_score, y=delta_test_mean)) +
geom_errorbar(aes(ymin=delta_test_mean-delta_test_sem, ymax=delta_test_mean+delta_test_sem)) +
geom_smooth(method="lm", se=T) +
geom_point(stat="identity", position="dodge", alpha=1, size=3, aes(color=training_vocab, shape=model)) +
ylab(metric) +
xlab("Syntax Generalization Score") +
ggtitle("Syntactic Generalization vs. Predictive Power") +
scale_color_manual(values = c("bllip-lg"="#440154FF",
"bllip-md"="#39568CFF",
"bllip-sm"="#1F968BFF",
"bllip-xs"="#73D055FF",
"gptbpe"="#888888")) +
facet_grid(~corpus, scales="free") +
theme(axis.text=element_text(size=14),
strip.text.x = element_text(size=14),
legend.text=element_text(size=14),
axis.title=element_text(size=18),
legend.position = "bottom")
#ggsave("./cogsci_images/sg_loglik.png",height=5,width=6)Regression analyses
We control for effects of perplexity by relating the residuals of a performance ~ PPL regression to SG score.
# Prepare a residualized regression for x1 onto y, controlling for the effects of x2.
d_resid = model_deltas %>%
drop_na() %>%
# Residualize delta metric w.r.t PPL for each model--training--seed
group_by(corpus) %>%
mutate(resid.delta = resid(lm(delta_test_mean ~ training:test_ppl))) %>%
ungroup() %>%
# Residualize SG score w.r.t. PPL for each training vocabulary
group_by(training_vocab) %>%
# NB no need for training:ppl interaction, since we're within-group.
mutate(resid.sg = resid(lm(sg_score ~ test_ppl))) %>%
ungroup() %>%
# Compute summary statistics across model--training--seed--corpus.
group_by(model, training_vocab, corpus, seed) %>%
summarise(resid.delta.mean = mean(resid.delta),
resid.delta.sem = sd(resid.delta) / sqrt(length(resid.delta)),
resid.sg.mean = mean(resid.sg),
resid.sg.sem = sd(resid.sg) / sqrt(length(resid.sg)))
# Now plot residual vs SG
d_resid %>%
#filter(corpus != "bnc-brown") %>%
ggplot(aes(x=resid.sg.mean, y=resid.delta.mean)) +
geom_errorbar(aes(xmin=resid.sg.mean - resid.sg.sem,
xmax=resid.sg.mean + resid.sg.sem,
ymin=resid.delta.mean - resid.delta.sem,
ymax=resid.delta.mean + resid.delta.sem), alpha=0.3) +
geom_smooth(method="lm", se=T) +
geom_point(stat="identity", position="dodge", alpha=1, size=4, aes(shape=model, color=training_vocab)) +
ylab(paste("Residual", metric)) +
xlab("Residual Syntax Generalization Score") +
ggtitle("Syntactic Generalization vs. Predictive Power") +
scale_color_manual(values = c("bllip-lg"="#440154FF",
"bllip-md"="#39568CFF",
"bllip-sm"="#1F968BFF",
"bllip-xs"="#73D055FF",
"gptbpe"="#888888")) +
facet_grid(.~corpus, scales="free") +
theme(axis.text=element_text(size=14),
strip.text.x = element_text(size=14),
legend.text=element_text(size=14),
axis.title=element_text(size=18),
legend.position = "right")Ignoring unknown aesthetics: xmin, xmaxggsave("../images/cuny2020/ppl_sg.png",height=4.5,width=11)
do_stepwise_regression = function(cur_corpus) {
regression_data = model_deltas %>%
filter(corpus == cur_corpus)
print("----------------------")
print(cur_corpus)
lm1 = lm(delta_test_mean ~ training_vocab:test_ppl, data = regression_data)
lm2 = lm(delta_test_mean ~ training_vocab:test_ppl + sg_score, data = regression_data)
print(anova(lm1, lm2))
summary(lm2)
}
#do_stepwise_regression("bnc-brown")
do_stepwise_regression("dundee")[1] "----------------------"
[1] "dundee"
Analysis of Variance Table
Model 1: delta_test_mean ~ training_vocab:test_ppl
Model 2: delta_test_mean ~ training_vocab:test_ppl + sg_score
Res.Df RSS Df Sum of Sq F Pr(>F)
1 23 0.00010177
2 22 0.00010139 1 3.7685e-07 0.0818 0.7776
Call:
lm(formula = delta_test_mean ~ training_vocab:test_ppl + sg_score,
data = regression_data)
Residuals:
Min 1Q Median 3Q Max
-0.002552 -0.001165 -0.000778 0.001200 0.003948
Coefficients:
Estimate Std. Error t value Pr(>|t|)
(Intercept) 7.197e-03 2.358e-03 3.052 0.00584 **
sg_score 9.602e-04 3.358e-03 0.286 0.77759
training_vocabbllip-lg:test_ppl -1.154e-05 2.185e-05 -0.528 0.60256
training_vocabbllip-md:test_ppl -1.842e-05 1.821e-05 -1.012 0.32263
training_vocabbllip-sm:test_ppl -2.326e-05 1.621e-05 -1.435 0.16545
training_vocabbllip-xs:test_ppl -2.323e-05 1.082e-05 -2.148 0.04299 *
training_vocabgptbpe:test_ppl -1.666e-06 5.315e-06 -0.313 0.75687
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
Residual standard error: 0.002147 on 22 degrees of freedom
Multiple R-squared: 0.3174, Adjusted R-squared: 0.1312
F-statistic: 1.705 on 6 and 22 DF, p-value: 0.1671do_stepwise_regression("natural-stories")[1] "----------------------"
[1] "natural-stories"
Analysis of Variance Table
Model 1: delta_test_mean ~ training_vocab:test_ppl
Model 2: delta_test_mean ~ training_vocab:test_ppl + sg_score
Res.Df RSS Df Sum of Sq F Pr(>F)
1 23 1.1819e-05
2 22 1.1817e-05 1 1.3684e-09 0.0025 0.9602
Call:
lm(formula = delta_test_mean ~ training_vocab:test_ppl + sg_score,
data = regression_data)
Residuals:
Min 1Q Median 3Q Max
-0.0012209 -0.0006400 0.0001431 0.0005430 0.0009429
Coefficients:
Estimate Std. Error t value Pr(>|t|)
(Intercept) 7.277e-03 8.050e-04 9.040 7.32e-09 ***
sg_score 5.786e-05 1.146e-03 0.050 0.960201
training_vocabbllip-lg:test_ppl -3.488e-05 7.458e-06 -4.676 0.000116 ***
training_vocabbllip-md:test_ppl -3.358e-05 6.216e-06 -5.403 2.00e-05 ***
training_vocabbllip-sm:test_ppl -3.445e-05 5.535e-06 -6.224 2.90e-06 ***
training_vocabbllip-xs:test_ppl -2.357e-05 3.693e-06 -6.383 2.01e-06 ***
training_vocabgptbpe:test_ppl -4.043e-06 1.814e-06 -2.228 0.036395 *
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
Residual standard error: 0.0007329 on 22 degrees of freedom
Multiple R-squared: 0.7971, Adjusted R-squared: 0.7417
F-statistic: 14.4 on 6 and 22 DF, p-value: 1.242e-06Predictive power and perplexity
model_deltas %>%
ggplot(aes(x=test_ppl, y=delta_test_mean, color=training_vocab)) +
geom_errorbar(aes(ymin=delta_test_mean-delta_test_sem, ymax=delta_test_mean+delta_test_sem), alpha=0.4) +
#geom_smooth(method="lm", se=F) +
geom_point(stat="identity", position="dodge", alpha=1, size=4, aes(shape=model)) +
ylab(metric) +
xlab("Test Perplexity") +
#coord_cartesian(ylim = c(1, 16)) +
ggtitle("Test Perplexity vs. Predictive Power") +
scale_color_manual(values = c("bllip-lg"="#440154FF",
"bllip-md"="#39568CFF",
"bllip-sm"="#1F968BFF",
"bllip-xs"="#73D055FF",
"gptbpe"="#888888")) +
facet_grid(~corpus, scales="free") +
#coord_cartesian(ylim = c(0, 150)) +
theme(axis.text=element_text(size=12),
strip.text.x = element_text(size=12),
legend.text=element_text(size=12),
axis.title=element_text(size=12),
legend.position = "right")
ggsave("../images/cuny2020/ppl_loglik.png",height=4.5,width=11)
Effect of training data size
model_deltas %>%
mutate(train_size = log(train_size)) %>%
ggplot(aes(x=train_size, y=delta_test_mean, color=model)) +
geom_errorbar(aes(ymin=delta_test_mean-delta_test_sem, ymax=delta_test_mean+delta_test_sem), width = 0.1) +
geom_smooth(method="lm", se=T, alpha=0.5) +
geom_point(stat="identity", position="dodge", alpha=1, size=3) +
ylab(metric) +
xlab("Log Million Training Tokens") +
ggtitle("Training Size vs. Predictive Power") +
facet_grid(corpus~model, scales="free") +
#scale_color_manual(values = c("#A42EF1", "#3894C8")) +
theme(axis.text=element_text(size=14),
strip.text.x = element_text(size=14),
legend.text=element_text(size=14),
axis.title=element_text(size=18),
legend.position = "bottom")
#ggsave("./cogsci_images/training_loglik.png",height=5,width=6)model_deltas %>%
mutate(train_size = log(train_size)) %>%
ggplot(aes(x=train_size, y=sg_score, color=model)) +
geom_smooth(method="lm", se=T, alpha=0.5) +
geom_point(stat="identity", position="dodge", alpha=1, size=3) +
ylab("SG SCore") +
xlab("Log Million Training Tokens") +
ggtitle("Training Size vs. Syntactic Generalization") +
#scale_color_manual(values = c("#A42EF1", "#3894C8")) +
facet_grid(~model, scales="free") +
theme(axis.text=element_text(size=14),
strip.text.x = element_text(size=14),
legend.text=element_text(size=14),
axis.title=element_text(size=18),
legend.position = "bottom")
#ggsave("./cogsci_images/training_sg.png",height=5,width=6)Smith & Levy reproduction
all_data %>%
#filter(surprisal < 15, surprisal > 0) %>%
mutate(bpe=str_detect(training, "bpe"),
training_source=str_replace(training, "-gptbpe", "")) %>%
ggplot(aes(x=surprisal, y=psychometric, color=training_source, linetype=bpe)) +
stat_smooth(se=T, alpha=0.5) +
#geom_errorbar(color="black", width=.2, position=position_dodge(width=.9), alpha=0.3) +
#geom_point(stat="identity", position="dodge", alpha=1, size=3) +
ylab("Processing Time (ms)") +
xlab("Surprisal (bits)") +
ggtitle("Surprisal vs. Reading Time / Gaze Duration") +
facet_grid(corpus~model, scales = "free") +
# scale_color_manual(values = c("bllip-lg"="#440154FF",
# "bllip-md"="#39568CFF",
# "bllip-sm"="#1F968BFF",
# "bllip-xs"="#73D055FF",
# "bllip-lg-gptbpe"="#888888",
# "bllip-md-gptbpe"="#888888",
# "bllip-sm-gptbpe"="#888888",
# "bllip-xs-gptbpe"="#888888")) +
theme(axis.text=element_text(size=14),
axis.text.y = element_text(size = 10),
strip.text.x = element_text(size=14),
legend.text=element_text(size=14),
axis.title=element_text(size=18),
legend.position = "right")
ggsave("../images/cuny2020/surp_corr.png",height=4.5,width=12)
Investigate vanilla
all_data %>%
#filter(surprisal < 15, surprisal > 0) %>%
filter(model == "vanilla") %>%
ggplot(aes(x=surprisal, y=psychometric)) +
#stat_smooth(se=T, alpha=0.5) +
#geom_errorbar(color="black", width=.2, position=position_dodge(width=.9), alpha=0.3) +
geom_point(alpha=0.1) + #stat="identity", position="dodge", alpha=1, size=3) +
ylab("Processing Time (ms)") +
xlab("Surprisal (bits)") +
ggtitle("Surprisal vs. Reading Time / Gaze Duration: Vanilla") +
facet_grid(corpus~training, scales = "free")
# scale_color_manual(values = c("bllip-lg"="#440154FF",
# "bllip-md"="#39568CFF",
# "bllip-sm"="#1F968BFF",
# "bllip-xs"="#73D055FF",
# "bllip-lg-gptbpe"="#888888",
# "bllip-md-gptbpe"="#888888",
# "bllip-sm-gptbpe"="#888888",
# "bllip-xs-gptbpe"="#888888"))all_data %>%
filter(corpus == "dundee", model == "vanilla", training == "bllip-lg", surprisal > 20, psychometric < 300)print(full_residuals %>% filter(corpus == "dundee", model == "vanilla", training == "bllip-lg") %>% arrange(desc(resid)))full_residuals %>% filter(corpus == "dundee", model == "vanilla", training == "bllip-lg") %>% arrange(desc(resid)) %>% filter(resid > 150) %>%
ggplot(aes(x=surprisal)) + geom_density()
Investigate RNNG
all_data %>%
#filter(surprisal < 15, surprisal > 0) %>%
filter(model == "rnng") %>%
ggplot(aes(x=surprisal, y=psychometric)) +
#stat_smooth(se=T, alpha=0.5) +
#geom_errorbar(color="black", width=.2, position=position_dodge(width=.9), alpha=0.3) +
geom_point(alpha=0.1) + #stat="identity", position="dodge", alpha=1, size=3) +
ylab("Processing Time (ms)") +
xlab("Surprisal (bits)") +
ggtitle("Surprisal vs. Reading Time / Gaze Duration: RNNG") +
facet_grid(corpus~training, scales = "free")
all_data %>%
filter(corpus == "dundee", model == "rnng", training == "bllip-lg", surprisal > 20, psychometric < 300)print(full_residuals %>% filter(corpus == "dundee", model == "rnng", training == "bllip-lg") %>% arrange(desc(resid)))full_residuals %>% filter(corpus == "dundee", model == "rnng", training == "bllip-lg") %>% arrange(desc(resid)) %>% filter(resid > 150) %>%
ggplot(aes(x=surprisal)) + geom_density()
Investigate ngram vs vanilla
ngram_resids = full_residuals %>% filter(model == "5gram", training == "bllip-sm") %>% group_by(corpus, code) %>% summarise(freq=mean(freq), psychometric=mean(psychometric), resid=mean(resid))
vanilla_resids = full_residuals %>% filter(model == "vanilla", training == "bllip-sm") %>% group_by(corpus, code) %>% summarise(freq=mean(freq), psychometric=mean(psychometric), resid=mean(resid))
resids_joined = ngram_resids %>% left_join(vanilla_resids, by=c("corpus", "code"), suffix=c(".ngram", ".vanilla"))
resids_joined %>%
ggplot(aes(x=resid.ngram, y=resid.vanilla)) + geom_point() + geom_abline(slope=1, color="red") +
facet_grid(~corpus)
resids_joined %>%
mutate(resid_abs_diff=abs(resid.ngram - resid.vanilla)) %>%
ggplot(aes(x=resid_abs_diff)) + geom_density() +
facet_grid(~corpus)
resids_joined %>%
mutate(resid_abs_diff=abs(resid.ngram - resid.vanilla)) %>%
ggplot(aes(x=freq.ngram, y=resid_abs_diff)) + geom_point(alpha=0.1) + geom_smooth()
Investigate gptbpe vs vanilla
gpt_resids = full_residuals %>% filter(model == "gpt2", training == "bllip-sm-gptbpe") %>% group_by(corpus, code) %>% summarise(freq=mean(freq), psychometric=mean(psychometric), resid=mean(resid))
vanilla_resids = full_residuals %>% filter(model == "vanilla", training == "bllip-sm") %>% group_by(corpus, code) %>% summarise(freq=mean(freq), psychometric=mean(psychometric), resid=mean(resid))
resids_joined = gpt_resids %>% left_join(vanilla_resids, by=c("corpus", "code"), suffix=c(".gpt", ".vanilla"))
resids_joined %>%
ggplot(aes(x=resid.gpt, y=resid.vanilla)) + geom_point() + geom_abline(slope=1, color="red") +
facet_grid(~corpus)
resids_joined %>%
mutate(resid_abs_diff=abs(resid.gpt - resid.vanilla)) %>%
ggplot(aes(x=resid_abs_diff)) + geom_density() +
facet_grid(~corpus)
resids_joined %>%
mutate(resid_abs_diff=abs(resid.gpt - resid.vanilla)) %>%
ggplot(aes(x=freq.gpt, y=resid_abs_diff)) + geom_point(alpha=0.1) + geom_smooth()
Investigate residuals overall
full_residuals %>% right_join(baseline_residuals, by=c("corpus", "code", "model", "training", "seed"), suffix=c(".full", ".baseline")) %>%
mutate(resid_delta=resid.full - resid.baseline,
training_source=as.factor(str_replace(training, "-gptbpe", "")),
bpe=str_detect(training, "gptbpe")) %>%
ggplot(aes(x=surprisal.full, y=resid_delta, color=training)) +
facet_grid(model~corpus) +
geom_point(alpha=0.1, size=0.5)
language_model_data %>% filter(model == "gpt2")LS0tCnRpdGxlOiAiQ1VOWSAyMDIwIEFuYWx5c2lzIgpvdXRwdXQ6IGh0bWxfbm90ZWJvb2sKLS0tCgojIFBhY2thZ2VzIGFuZCB1dGlsaXRpZXMKCmBgYHtyfQpsaWJyYXJ5KHRpZHl2ZXJzZSkKbGlicmFyeShsbWU0KQpsaWJyYXJ5KGxtZXJUZXN0KQpsaWJyYXJ5KGxvZ2dpbmcpCmxpYnJhcnkobXZ0bm9ybSkKbGlicmFyeShtZ2N2KQpgYGAKCmBgYHtyfQojIENvbXB1dGUgdGhlIGxvZy1saWtlbGlob29kIG9mIGEgbmV3IGRhdGFzZXQgdXNpbmcgYSBmaXQgbG1lNCBtb2RlbC4KbG9nTGlrX3Rlc3QgPC0gZnVuY3Rpb24obG0sIHRlc3RfWCwgdGVzdF95KSB7CiAgcHJlZGljdGlvbnMgPC0gcHJlZGljdChsbSwgdGVzdF9YLCByZS5mb3JtPU5BKQogICMgR2V0IHN0ZC5kZXYuIG9mIHJlc2lkdWFsLCBlc3RpbWF0ZWQgZnJvbSB0cmFpbiBkYXRhCiAgc3RkZXYgPC0gc2lnbWEobG0pCiAgIyBGb3IgZWFjaCBwcmVkaWN0aW9uLS1vYnNlcnZhdGlvbiwgZ2V0IHRoZSBkZW5zaXR5IHAob2JzIHwgTihwcmVkaWN0ZWQsIG1vZGVsX3NpZ21hKSkgYW5kIHJlZHVjZQogIGRlbnNpdHkgPC0gc3VtKGRub3JtKHRlc3RfeSwgcHJlZGljdGlvbnMsIHN0ZGV2LCBsb2c9VFJVRSkpCiAgcmV0dXJuKGRlbnNpdHkpCn0KIyBHZXQgcGVyLXByZWRpY3Rpb24gbG9nLWxpa2VsaWhvb2QKbG9nTGlrX3Rlc3RfcGVyIDwtIGZ1bmN0aW9uKGxtLCB0ZXN0X1gsIHRlc3RfeSkgewogIHByZWRpY3Rpb25zIDwtIHByZWRpY3QobG0sIHRlc3RfWCwgcmUuZm9ybT1OQSkKICAjIEdldCBzdGQuZGV2LiBvZiByZXNpZHVhbCwgZXN0aW1hdGVkIGZyb20gdHJhaW4gZGF0YQogIHN0ZGV2IDwtIHNpZ21hKGxtKQogICMgRm9yIGVhY2ggcHJlZGljdGlvbi0tb2JzZXJ2YXRpb24sIGdldCB0aGUgZGVuc2l0eSBwKG9icyB8IE4ocHJlZGljdGVkLCBtb2RlbF9zaWdtYSkpCiAgZGVuc2l0aWVzIDwtIGRub3JtKHRlc3RfeSwgcHJlZGljdGlvbnMsIHN0ZGV2LCBsb2c9VFJVRSkKICByZXR1cm4oZGVuc2l0aWVzKQp9CiMgQ29tcHV0ZSBNU0Ugb2YgYSBuZXcgZGF0YXNldCB1c2luZyBhIGZpdCBsbWU0IG1vZGVsLgptc2VfdGVzdCA8LSBmdW5jdGlvbihsbSwgdGVzdF9YLCB0ZXN0X3kpIHsKICByZXR1cm4obWVhbigocHJlZGljdChsbSwgdGVzdF9YLCByZS5mb3JtPU5BKSAtIHRlc3RfeSkgXiAyKSkKfQojU2FuaXR5IGNoZWNrcwojbXlsbSA8LSBnYW0ocHN5Y2hvbWV0cmljIH4gIHMoc3VycHJpc2FsLCBicyA9ICJjciIsIGsgPSAyMCkgKyBzKHByZXZfc3VycCwgYnMgPSAiY3IiLCBrID0gMjApICsgdGUoZnJlcSwgbGVuLCBicyA9ICJjciIpICsgdGUocHJldl9mcmVxLCBwcmV2X2xlbiwgYnMgPSAiY3IiKSwgZGF0YT10cmFpbl9kYXRhKQojYyhsb2dMaWsobXlsbSksIGxvZ0xpa190ZXN0KG15bG0sIHRyYWluX2RhdGEsIHRyYWluX2RhdGEkcHN5Y2hvbWV0cmljKSkKI2xvZ0xpa190ZXN0KG15bG0sIHRlc3RfZGF0YSwgdGVzdF9kYXRhJHBzeWNob21ldHJpYykKYGBgCgojIERhdGEgbG9hZGluZyBhbmQgcHJlcHJvY2Vzc2luZwoKYGBge3IgTG9hZCBhbmQgcHJlcHJvY2VzcyBkYXRhfQpkYXRhID0gcmVhZC5jc3YoIi4uL2RhdGEvaGFybW9uaXplZF9yZXN1bHRzLmNzdiIpCgphbGxfZGF0YSA9IGRhdGEgJT4lCiAgbXV0YXRlKHNlZWQgPSBhcy5mYWN0b3Ioc2VlZCkpICU+JQogIGdyb3VwX2J5KGNvcnB1cywgbW9kZWwsIHRyYWluaW5nLCBzZWVkKSAlPiUKICAgIG11dGF0ZShwcmV2X3N1cnAgPSBsYWcoc3VycHJpc2FsKSwKICAgICAgICAgcHJldl9jb2RlID0gbGFnKGNvZGUpLAogICAgICAgICBwcmV2X2xlbiA9IGxhZyhsZW4pLAogICAgICAgICBwcmV2X2ZyZXEgPSBsYWcoZnJlcSksCiAgICAgICAgIHByZXZfc3VycCA9IGxhZyhzdXJwcmlzYWwpLAogICAgICAgICAKICAgICAgICAgcHJldjJfZnJlcSA9IGxhZyhwcmV2X2ZyZXEpLAogICAgICAgICBwcmV2Ml9jb2RlID0gbGFnKHByZXZfY29kZSksCiAgICAgICAgIHByZXYyX2xlbiA9IGxhZyhwcmV2X2xlbiksCiAgICAgICAgIHByZXYyX3N1cnAgPSBsYWcocHJldl9zdXJwKSwKICAgICAgICAgCiAgICAgICAgIHByZXYzX2ZyZXEgPSBsYWcocHJldjJfZnJlcSksCiAgICAgICAgIHByZXYzX2NvZGUgPSBsYWcocHJldjJfY29kZSksCiAgICAgICAgIHByZXYzX2xlbiA9IGxhZyhwcmV2Ml9sZW4pLAogICAgICAgICBwcmV2M19zdXJwID0gbGFnKHByZXYyX3N1cnApKSAlPiUKICB1bmdyb3VwKCkgJT4lCiAgCiAgIyBGaWx0ZXIgYmFjayB0aHJlZSBmb3IgdGhlIGR1bmRlZSBjb3JwdXMuIEZpbHRlciBiYWNrIDEgZm9yIGFsbCBvdGhlciBjb3Jwb3JhCiAgIyBOQiB0aGlzIGVmZmVjdGl2ZWx5IHJlbW92ZXMgYWxsIHplcm8tc3VycHJpc2FsIHJvd3MsIHNpbmNlIGVhcmx5LXNlbnRlbmNlIHRva2VucyBkb24ndCBoYXZlIGNvbnRpZ3VvdXMgdG9rZW4gaGlzdG9yeQogIGZpbHRlcigoY29ycHVzID09ICJkdW5kZWUiICYgY29kZSA9PSBwcmV2Ml9jb2RlICsgMikgfCAoY29ycHVzICE9ICJkdW5kZWUiICYgY29kZSA9PSBwcmV2X2NvZGUgKyAxKSkgJT4lCiAgCiAgc2VsZWN0KC1wcmV2X2NvZGUsIC1wcmV2Ml9jb2RlLCAtcHJldjNfY29kZSkgJT4lCiAgZHJvcF9uYSgpCgphbGxfZGF0YSA9IGFsbF9kYXRhICU+JQogIG11dGF0ZSgKICAgIG1vZGVsID0gYXMuY2hhcmFjdGVyKG1vZGVsKSwKICAgIG1vZGVsID0gaWZfZWxzZShtb2RlbCA9PSAiZ3B0LTIiLCAiZ3B0MiIsIG1vZGVsKSwKICAgIG1vZGVsID0gYXMuZmFjdG9yKG1vZGVsKSkKYGBgCgpgYGB7cn0KbWlzc2luZ19yb3dzID0gYWxsX2RhdGEgJT4lIGNvbXBsZXRlKG5lc3RpbmcoY29ycHVzLCBjb2RlKSwgbmVzdGluZyhtb2RlbCwgdHJhaW5pbmcsIHNlZWQpKSAlPiUgCiAgZ3JvdXBfYnkoY29ycHVzLCBjb2RlKSAlPiUgCiAgICBmaWx0ZXIoc3VtKGlzLm5hKHN1cnByaXNhbCkpID4gMCkgJT4lIAogIHVuZ3JvdXAoKSAlPiUgCiAgYW50aV9qb2luKGFsbF9kYXRhLCBieT1jKCJjb3JwdXMiLCAiY29kZSIsICJtb2RlbCIsICJ0cmFpbmluZyIsICJzZWVkIikpCgptaXNzaW5nX3Jvd3MgJT4lIGdncGxvdChhZXMoeD1jb3JwdXMsIGZpbGw9ZmFjdG9yKHBhc3RlKG1vZGVsLHRyYWluaW5nKSkpKSArIGdlb21fYmFyKHBvc2l0aW9uPXBvc2l0aW9uX2RvZGdlKHdpZHRoPTAuOCkpCnByaW50KG1pc3Npbmdfcm93cyAlPiUgZ3JvdXBfYnkobW9kZWwsIHRyYWluaW5nLCBzZWVkLCBjb3JwdXMpICU+JSBzdW1tYXJpc2Uobj1uKCkpKSAlPiUgYXJyYW5nZShkZXNjKG4pKQpgYGAKCgpgYGB7ciBEcm9wIHRva2VucyBmb3Igd2hpY2ggYW55IG1vZGVsIGlzIG1pc3Npbmcgc3VycHJpc2FsIGRhdGEufQoKIyBDb21wdXRlIHRoZSBpZGVhbCBudW1iZXIgb2YgbW9kZWwtLXNlZWQtLXRyYWluaW5nIG9ic2VydmF0aW9ucyBwZXIgdG9rZW4uCnRvX2Ryb3AgPSBhbGxfZGF0YSAlPiUKICBncm91cF9ieShjb3JwdXMsIGNvZGUpICU+JSBzdW1tYXJpc2UobiA9IG4oKSkgJT4lIHVuZ3JvdXAoKSAlPiUKICBncm91cF9ieShjb3JwdXMpICU+JSBtdXRhdGUoIG1heF9uID0gbWF4KG4pKSAlPiUgdW5ncm91cCgpICU+JQogIGZpbHRlcihtYXhfbiAhPSBuKSAlPiUKICBzZWxlY3QoY29kZSwgY29ycHVzKQoKI3RvX2Ryb3AgPSBhbGxfZGF0YSAlPiUgZ3JvdXBfYnkoY29ycHVzLCBjb2RlKSAlPiUgZmlsdGVyKG4oKSAhPSBpZGVhbF90b2tlbl9vYnNfY291bnQpICU+JSB1bmdyb3VwKCkKbG9naW5mbyhwYXN0ZSgiRHJvcHBpbmciLCBucm93KHRvX2Ryb3ApLCAib2JzZXJ2YXRpb25zIGNvcnJlc3BvbmRpbmcgdG8gY29ycHVzIHRva2VucyB3aGljaCBhcmUgbWlzc2luZyBvYnNlcnZhdGlvbnMgZm9yIHNvbWUgbW9kZWwuIikpCmxvZ2luZm8ocGFzdGUoIkRyb3BwaW5nIiwgdG9fZHJvcCAlPiUgZ3JvdXBfYnkoY29ycHVzLCBjb2RlKSAlPiUgbl9ncm91cHMoKSwgInRva2VucyB3aGljaCBhcmUgbWlzc2luZyBvYnNlcnZhdGlvbnMgZm9yIHNvbWUgbW9kZWwuIikpCgphbGxfZGF0YSA9IGFsbF9kYXRhICU+JSBhbnRpX2pvaW4odG9fZHJvcCAlPiUgZ3JvdXBfYnkoY29ycHVzLCBjb2RlKSwgYnk9YygiY29ycHVzIiwgImNvZGUiKSkKbG9naW5mbyhwYXN0ZSgiQWZ0ZXIgZHJvcCwiLCBucm93KGFsbF9kYXRhKSwgIm9ic2VydmF0aW9ucyAoIiwgYWxsX2RhdGEgJT4lIGdyb3VwX2J5KGNvcnB1cywgY29kZSkgJT4lIG5fZ3JvdXBzKCksICIgdG9rZW5zKSByZW1haW4uIikpCmBgYAoKYGBge3IgRHJvcCB0b2tlbnMgZm9yIHdoaWNoIGFueSBtb2RlbCBoYXMgemVyby12YWx1ZWQgc3VycHJpc2Fscy59Cgp0b19kcm9wX3plcm9fc3VycHMgPSBhbGxfZGF0YSAlPiUgZ3JvdXBfYnkoY29ycHVzLCBjb2RlKSAlPiUgZmlsdGVyKGFueShzdXJwcmlzYWwgPT0gMCkpICU+JSB1bmdyb3VwKCkKbG9naW5mbyhwYXN0ZSgiRHJvcHBpbmciLCBucm93KHRvX2Ryb3BfemVyb19zdXJwcyksICJvYnNlcnZhdGlvbnMgY29ycmVzcG9uZGluZyB0byBjb3JwdXMgdG9rZW5zIHdoaWNoIGhhdmUgc3VycHJpc2FsIHplcm9zIGZvciBzb21lIG1vZGVsLiIpKQpsb2dpbmZvKHBhc3RlKCJEcm9wcGluZyIsIHRvX2Ryb3BfemVyb19zdXJwcyAlPiUgZ3JvdXBfYnkoY29ycHVzLCBjb2RlKSAlPiUgbl9ncm91cHMoKSwgInRva2VucyB3aGljaCBoYXZlIHN1cnByaXNhbCB6ZXJvcyBmb3Igc29tZSBtb2RlbC4iKSkKCmFsbF9kYXRhID0gYWxsX2RhdGEgJT4lIGFudGlfam9pbih0b19kcm9wX3plcm9fc3VycHMgJT4lIGdyb3VwX2J5KGNvcnB1cywgY29kZSksIGJ5PWMoImNvcnB1cyIsICJjb2RlIikpCmxvZ2luZm8ocGFzdGUoIkFmdGVyIGRyb3AsIiwgbnJvdyhhbGxfZGF0YSksICJvYnNlcnZhdGlvbnMgKCIsIGFsbF9kYXRhICU+JSBncm91cF9ieShjb3JwdXMsIGNvZGUpICU+JSBuX2dyb3VwcygpLCAiIHRva2VucykgcmVtYWluLiIpKQpgYGAKCmBgYHtyIERyb3AgdG9rZW5zIGZvciB3aGljaCB3ZSBoYXZlIHplcm8tdmFsdWVkIHBzeWNob21ldHJpYyBkYXRhLn0KCnRvX2Ryb3BfemVyb19wc3ljaHMgPSBhbGxfZGF0YSAlPiUgZ3JvdXBfYnkoY29ycHVzLCBjb2RlKSAlPiUgZmlsdGVyKGFueShwc3ljaG9tZXRyaWMgPT0gMCkpICU+JSB1bmdyb3VwKCkKbG9naW5mbyhwYXN0ZSgiRHJvcHBpbmciLCBucm93KHRvX2Ryb3BfemVyb19wc3ljaHMpLCAib2JzZXJ2YXRpb25zIGNvcnJlc3BvbmRpbmcgdG8gY29ycHVzIHRva2VucyB3aGljaCBoYXZlIHN1cnByaXNhbCB6ZXJvcyBmb3Igc29tZSBtb2RlbC4iKSkKbG9naW5mbyhwYXN0ZSgiRHJvcHBpbmciLCB0b19kcm9wX3plcm9fcHN5Y2hzICU+JSBncm91cF9ieShjb3JwdXMsIGNvZGUpICU+JSBuX2dyb3VwcygpLCAidG9rZW5zIHdoaWNoIGhhdmUgc3VycHJpc2FsIHplcm9zIGZvciBzb21lIG1vZGVsLiIpKQoKYWxsX2RhdGEgPSBhbGxfZGF0YSAlPiUgYW50aV9qb2luKHRvX2Ryb3BfemVyb19wc3ljaHMgJT4lIGdyb3VwX2J5KGNvcnB1cywgY29kZSksIGJ5PWMoImNvcnB1cyIsICJjb2RlIikpCmxvZ2luZm8ocGFzdGUoIkFmdGVyIGRyb3AsIiwgbnJvdyhhbGxfZGF0YSksICJvYnNlcnZhdGlvbnMgKCIsIGFsbF9kYXRhICU+JSBncm91cF9ieShjb3JwdXMsIGNvZGUpICU+JSBuX2dyb3VwcygpLCAiIHRva2VucykgcmVtYWluLiIpKQpgYGAKCiAKIyBMZWFybiBtb2RlbHMKIApgYGB7cn0KIyBDb21wdXRlIGxpbmVhciBtb2RlbCBzdGF0cyBmb3IgdGhlIGdpdmVuIHRyYWluaW5nIGRhdGEgc3Vic2V0IGFuZCBmdWxsIHRlc3QgZGF0YS4KIyBBdXRvbWF0aWNhbGx5IHN1YnNldHMgdGhlIHRlc3QgZGF0YSB0byBtYXRjaCB0aGUgcmVsZXZhbnQgZ3JvdXAgZm9yIHdoaWNoIHdlIGFyZSB0cmFpbmluZyBhIGxpbmVhciBtb2RlbC4KZ2V0X2xtX2RhdGEgPC0gZnVuY3Rpb24oZGYsIHRlc3RfZGF0YSwgZm9ybXVsYSwgZm9sZCwgc3RvcmVfZW52KSB7CiAgI3RoaXNfbG0gPC0gZ2FtKGZvcm11bGEsIGRhdGE9ZGYpOwogIHRoaXNfbG0gPSBsbShmb3JtdWxhLCBkYXRhPWRmKQogIHRoaXNfdGVzdF9kYXRhIDwtIHNlbWlfam9pbih0ZXN0X2RhdGEsIGRmLCBieT1jKCJ0cmFpbmluZyIsICJtb2RlbCIsICJzZWVkIiwgImNvcnB1cyIpKTsKICAKICAjIFNhdmUgbG0gdG8gdGhlIGdsb2JhbCBlbnYgc28gdGhhdCB3ZSBjYW4gYWNjZXNzIHJlc2lkdWFscyBsYXRlci4KICBsbV9uYW1lID0gcGFzdGUodW5pcXVlKHBhc3RlKGRmJG1vZGVsLCBkZiR0cmFpbmluZywgZGYkc2VlZCwgZGYkY29ycHVzKSlbMV0sIGZvbGQpCiAgYXNzaWduKGxtX25hbWUsIHRoaXNfbG0sIGVudmlyPXN0b3JlX2VudikKICAKICBzdW1tYXJpc2UoZGYsCiAgICAgICAgICAgIGxvZ19saWsgPSBhcy5udW1lcmljKGxvZ0xpayh0aGlzX2xtLCBSRU1MID0gRikpLAogICAgICAgICAgICB0ZXN0X2xpayA9IGxvZ0xpa190ZXN0KHRoaXNfbG0sIHRoaXNfdGVzdF9kYXRhLCB0aGlzX3Rlc3RfZGF0YSRwc3ljaG9tZXRyaWMpLAogICAgICAgICAgICB0ZXN0X21zZSA9IG1zZV90ZXN0KHRoaXNfbG0sIHRoaXNfdGVzdF9kYXRhLCB0aGlzX3Rlc3RfZGF0YSRwc3ljaG9tZXRyaWMpKQp9CiMgRm9yIGEgcHJldmlvdXNseSBmaXR0ZWQgbG0gc3RvcmVkIGluIHN0b3JlX2VudiwgZ2V0IHRoZSByZXNpZHVhbHMgb24gdGVzdCBkYXRhIG9mIHRoZSByZWxldmFudCBkYXRhIHN1YnNldC4KZ2V0X2xtX3Jlc2lkdWFscyA8LSBmdW5jdGlvbihkZiwgZm9sZCwgc3RvcmVfZW52KSB7CiAgIyBSZXRyaWV2ZSB0aGUgcmVsZXZhbnQgbG0uCiAgbG1fbmFtZSA9IHBhc3RlKHVuaXF1ZShwYXN0ZShkZiRtb2RlbCwgZGYkdHJhaW5pbmcsIGRmJHNlZWQsIGRmJGNvcnB1cykpWzFdLCBmb2xkKQogIHRoaXNfbG0gPC0gZ2V0KGxtX25hbWUsIGVudmlyPXN0b3JlX2VudikKICAKICBtdXRhdGUoZGYsCiAgICAgICAgIGxpa2VsaWhvb2QgPSBsb2dMaWtfdGVzdF9wZXIodGhpc19sbSwgZGYsIGRmJHBzeWNob21ldHJpYyksCiAgICAgICAgIHJlc2lkID0gZGYkcHN5Y2hvbWV0cmljIC0gcHJlZGljdCh0aGlzX2xtLCBkZiwgcmUuZm9ybT1OQSkpCn0KIyBDb21wdXRlIHBlci1leGFtcGxlIGRlbHRhLWxvZy1saWtlbGlob29kIGZvciB0aGUgZ2l2ZW4gdGVzdCBmb2xkLgpnZXRfbG1fZGVsdGFfbG9nX2xpayA8LSBmdW5jdGlvbih0ZXN0X2RhdGEsIGZvbGQsIGJhc2VsaW5lX2VudiwgZnVsbF9lbnYpIHsKICBsbV9uYW1lID0gcGFzdGUodW5pcXVlKHBhc3RlKHRlc3RfZGF0YSRtb2RlbCwgdGVzdF9kYXRhJHRyYWluaW5nLCB0ZXN0X2RhdGEkc2VlZCwgdGVzdF9kYXRhJGNvcnB1cykpWzFdLCBmb2xkKQogIGJhc2VsaW5lX2xtIDwtIGdldChsbV9uYW1lLCBlbnZpcj1iYXNlbGluZV9lbnYpCiAgZnVsbF9sbSA8LSBnZXQobG1fbmFtZSwgZW52aXI9ZnVsbF9lbnYpCiAgCiAgZGVsdGFfbG9nX2xpayA9IGxvZ0xpa190ZXN0X3BlcihmdWxsX2xtLCB0ZXN0X2RhdGEsIHRlc3RfZGF0YSRwc3ljaG9tZXRyaWMpIC0gbG9nTGlrX3Rlc3RfcGVyKGJhc2VsaW5lX2xtLCB0ZXN0X2RhdGEsIHRlc3RfZGF0YSRwc3ljaG9tZXRyaWMpCiAgcmV0dXJuKGNiaW5kKHRlc3RfZGF0YSwgZGVsdGFfbG9nX2xpaz1kZWx0YV9sb2dfbGlrKSkKfQojIyMjIwojIERlZmluZSByZWdyZXNzaW9uIGZvcm11bGFlLgojIEV5ZS10cmFja2luZyByZWdyZXNzaW9uOiBvbmx5IHVzZSBzdXJwcmlzYWwgYW5kIHByZXZpb3VzIHN1cnByaXNhbDsgU1BSVCByZWdyZXNzaW9uOiB1c2UgMi1iYWNrIGZlYXR1cmVzLgojYmFzZWxpbmVfcnRfcmVncmVzc2lvbiA9IHBzeWNob21ldHJpYyB+IHRlKGZyZXEsIGxlbiwgYnMgPSAiY3IiKSArIHRlKHByZXZfZnJlcSwgcHJldl9sZW4sIGJzID0gImNyIikKI2Jhc2VsaWVfc3BydF9yZWdyZXNzaW9uID0gcHN5Y2hvbWV0cmljIH4gdGUoZnJlcSwgbGVuLCBicyA9ICJjciIpICsgdGUocHJldl9mcmVxLCBwcmV2X2xlbiwgYnMgPSAiY3IiKSArIHRlKHByZXYyX2ZyZXEsIHByZXYyX2xlbiwgYnMgPSAiY3IiKQojZnVsbF9ydF9yZWdyZXNzaW9uID0gKHBzeWNob21ldHJpYyB+IHMoc3VycHJpc2FsLCBicyA9ICJjciIsIGsgPSAyMCkgKyBzKHByZXZfc3VycCwgYnMgPSAiY3IiLCBrID0gMjApCiAgICAgICAgICAgICAgICAgICAgICMrIHRlKGZyZXEsIGxlbiwgYnMgPSAiY3IiKSArIHRlKHByZXZfZnJlcSwgcHJldl9sZW4sIGJzID0gImNyIikpCiNmdWxsX3NwcnRfcmVncmVzc2lvbiA9IChwc3ljaG9tZXRyaWMgfiBzKHN1cnByaXNhbCwgYnMgPSAiY3IiLCBrID0gMjApICsgcyhwcmV2X3N1cnAsIGJzID0gImNyIiwgayA9IDIwKSArIHMocHJldjJfc3VycCwgYnMgPSAiY3IiLCBrID0gMjApCiAgICAgICAgICAgICAgICAgICAgICAgICMrIHRlKGZyZXEsIGxlbiwgYnMgPSAiY3IiKSArIHRlKHByZXZfZnJlcSwgcHJldl9sZW4sIGJzID0gImNyIikgKyB0ZShwcmV2Ml9mcmVxLCBwcmV2Ml9sZW4sIGJzID0gImNyIikpCgpiYXNlbGluZV9ydF9yZWdyZXNzaW9uID0gcHN5Y2hvbWV0cmljIH4gZnJlcSArIHByZXZfZnJlcSArIHByZXYyX2ZyZXEgKyBwcmV2M19mcmVxICsgbGVuICsgcHJldl9sZW4gKyBwcmV2Ml9sZW4gKyBwcmV2M19sZW4KYmFzZWxpbmVfc3BydF9yZWdyZXNzaW9uID0gcHN5Y2hvbWV0cmljIH4gZnJlcSArIHByZXZfZnJlcSArIGxlbiArIHByZXZfbGVuCgpmdWxsX3J0X3JlZ3Jlc3Npb24gPSBwc3ljaG9tZXRyaWMgfiBzdXJwcmlzYWwgKyBwcmV2X3N1cnAgKyBwcmV2Ml9zdXJwICsgcHJldjNfc3VycCArIGZyZXEgKyBwcmV2X2ZyZXEgKyBwcmV2Ml9mcmVxICsgcHJldjNfZnJlcSArIGxlbiArIHByZXZfbGVuICsgcHJldjJfbGVuICsgcHJldjNfbGVuCmZ1bGxfc3BydF9yZWdyZXNzaW9uID0gcHN5Y2hvbWV0cmljIH4gc3VycHJpc2FsICsgcHJldl9zdXJwICsgZnJlcSArIHByZXZfZnJlcSArIGxlbiArIHByZXZfbGVuCiAgCiMjIyMjCiMgUHJlcGFyZSBmcmFtZXMvZW52aXJvbm1lbnRzIGZvciBzdG9yaW5nIHJlc3VsdHMvb2JqZWN0cy4KYmFzZWxpbmVfcmVzdWx0cyA9IGRhdGEuZnJhbWUoKQpmdWxsX21vZGVsX3Jlc3VsdHMgPSBkYXRhLmZyYW1lKCkKYmFzZWxpbmVfcmVzaWR1YWxzID0gZGF0YS5mcmFtZSgpCmZ1bGxfcmVzaWR1YWxzID0gZGF0YS5mcmFtZSgpCmxvZ19saWtfZGVsdGFzID0gZGF0YS5mcmFtZSgpCgojUmFuZG9tbHkgc2h1ZmZsZSB0aGUgZGF0YQphbGxfZGF0YTwtYWxsX2RhdGFbc2FtcGxlKG5yb3coYWxsX2RhdGEpKSxdCiNDcmVhdGUgSyBlcXVhbGx5IHNpemUgZm9sZHMKSyA9IDUKZm9sZHMgPC0gY3V0KHNlcSgxLG5yb3coYWxsX2RhdGEpKSxicmVha3M9SyxsYWJlbHM9RkFMU0UpCiNQZXJmb3JtIDEwIGZvbGQgY3Jvc3MgdmFsaWRhdGlvbgoKIyBGaXQgbW9kZWxzIGZvciBzb21lIGZvbGQgb2YgdGhlIGRhdGEuCmJhc2VsaW5lX2NvcnB1cyA9IGZ1bmN0aW9uKGNvcnB1cywgZGYsIHRlc3RfZGF0YSwgZm9sZCwgZW52KSB7CiAgaWYoY29ycHVzID09ICJkdW5kZWUiKSB7CiAgICBnZXRfbG1fZGF0YShkZiwgdGVzdF9kYXRhLCBiYXNlbGluZV9ydF9yZWdyZXNzaW9uLCBmb2xkLCBlbnYpCiAgfSBlbHNlIHsKICAgIGdldF9sbV9kYXRhKGRmLCB0ZXN0X2RhdGEsIGJhc2VsaW5lX3NwcnRfcmVncmVzc2lvbiwgZm9sZCwgZW52KQogIH0KfQpmdWxsX21vZGVsX2NvcnB1cyA9IGZ1bmN0aW9uKGNvcnB1cywgZGYsIHRlc3RfZGF0YSwgZm9sZCwgZW52KSB7CiAgaWYoY29ycHVzWzFdID09ICJkdW5kZWUiKSB7CiAgICBnZXRfbG1fZGF0YShkZiwgdGVzdF9kYXRhLCBmdWxsX3J0X3JlZ3Jlc3Npb24sIGZvbGQsIGVudikKICB9IGVsc2UgewogICAgZ2V0X2xtX2RhdGEoZGYsIHRlc3RfZGF0YSwgZnVsbF9zcHJ0X3JlZ3Jlc3Npb24sIGZvbGQsIGVudikKICB9Cn0KCiMgUHJlcGFyZSBhIG5ldyBFbnZpcm9ubWVudCBpbiB3aGljaCB3ZSBzdG9yZSBmaXR0ZWQgTE1zLCB3aGljaCB3ZSdsbCBxdWVyeSBsYXRlciBmb3IgcmVzaWR1YWxzIGFuZCBvdGhlciBtZXRyaWNzLgpiYXNlbGluZV9lbnYgPSBuZXcuZW52KCkKZnVsbF9lbnYgPSBuZXcuZW52KCkKCmZvcihpIGluIDE6SykgeyAKICAjU2VnZW1lbnQgeW91ciBkYXRhIGJ5IGZvbGQgdXNpbmcgdGhlIHdoaWNoKCkgZnVuY3Rpb24gCiAgdGVzdEluZGV4ZXMgPC0gd2hpY2goZm9sZHM9PWksIGFyci5pbmQ9VFJVRSkKICB0ZXN0X2RhdGEgPC0gYWxsX2RhdGFbdGVzdEluZGV4ZXMsIF0KICB0cmFpbl9kYXRhIDwtIGFsbF9kYXRhWy10ZXN0SW5kZXhlcywgXQogIAogICMgQ29tcHV0ZSBhIGJhc2VsaW5lIGxpbmVhciBtb2RlbCBmb3IgZWFjaCBtb2RlbC0tdHJhaW5pbmctLXNlZWQtLVJULWNvcnB1cyBjb21iaW5hdGlvbi4KICBiYXNlbGluZXMgPSB0cmFpbl9kYXRhICU+JQogICAgZ3JvdXBfYnkobW9kZWwsIHRyYWluaW5nLCBzZWVkLCBjb3JwdXMpICU+JQogICAgICBwcmludChtb2RlbCkgJT4lCiAgICAgIGRvKGJhc2VsaW5lX2NvcnB1cyh1bmlxdWUoLiRjb3JwdXMpLCAuLCB0ZXN0X2RhdGEsIGksIGJhc2VsaW5lX2VudikpICU+JQogICAgdW5ncm91cCgpICU+JQogICAgbXV0YXRlKHNlZWQgPSBhcy5mYWN0b3Ioc2VlZCksCiAgICAgICAgICAgZm9sZCA9IGkpCiAgCiAgYmFzZWxpbmVfcmVzdWx0cyA9IHJiaW5kKGJhc2VsaW5lX3Jlc3VsdHMsIGJhc2VsaW5lcykKICAKICAjIENvbXB1dGUgYSBmdWxsIGxpbmVhciBtb2RlbCBmb3IgZWFjaCBtb2RlbC0tdHJhaW5pbmctLXNlZWQtUlQtY29ycHVzIGNvbWJpbmF0aW9uCiAgZnVsbF9tb2RlbHMgPSB0cmFpbl9kYXRhICU+JQogICAgZ3JvdXBfYnkobW9kZWwsIHRyYWluaW5nLCBzZWVkLCBjb3JwdXMpICU+JQogICAgICBkbyhmdWxsX21vZGVsX2NvcnB1cyh1bmlxdWUoLiRjb3JwdXMpLCAuLCB0ZXN0X2RhdGEsIGksIGZ1bGxfZW52KSkgJT4lCiAgICB1bmdyb3VwKCkgJT4lCiAgICBtdXRhdGUoc2VlZCA9IGFzLmZhY3RvcihzZWVkKSwKICAgICAgICAgICBmb2xkID0gaSkKICAKICBmdWxsX21vZGVsX3Jlc3VsdHMgPSByYmluZChmdWxsX21vZGVsX3Jlc3VsdHMsIGZ1bGxfbW9kZWxzKQogIAogICMgQ29tcHV0ZSBkZWx0YS1sb2ctbGlrZWxpaG9vZHMKICBmb2xkX2xvZ19saWtfZGVsdGFzID0gdGVzdF9kYXRhICU+JQogICAgZ3JvdXBfYnkobW9kZWwsIHRyYWluaW5nLCBzZWVkLCBjb3JwdXMpICU+JQogICAgICBkbyhnZXRfbG1fZGVsdGFfbG9nX2xpayguLCBpLCBiYXNlbGluZV9lbnYsIGZ1bGxfZW52KSkgJT4lCiAgICB1bmdyb3VwKCkKCiAgbG9nX2xpa19kZWx0YXMgPSByYmluZChsb2dfbGlrX2RlbHRhcywgZm9sZF9sb2dfbGlrX2RlbHRhcykKICAKICBmb2xkX2Jhc2VsaW5lX3Jlc2lkdWFscyA9IHRlc3RfZGF0YSAlPiUKICAgIGdyb3VwX2J5KG1vZGVsLCB0cmFpbmluZywgc2VlZCwgY29ycHVzKSAlPiUKICAgICAgZG8oZ2V0X2xtX3Jlc2lkdWFscyguLCBpLCBiYXNlbGluZV9lbnYpKSAlPiUKICAgIHVuZ3JvdXAoKQoKICBiYXNlbGluZV9yZXNpZHVhbHMgPSByYmluZChiYXNlbGluZV9yZXNpZHVhbHMsIGZvbGRfYmFzZWxpbmVfcmVzaWR1YWxzKQoKICBmb2xkX2Z1bGxfcmVzaWR1YWxzID0gdGVzdF9kYXRhICU+JQogICAgZ3JvdXBfYnkobW9kZWwsIHRyYWluaW5nLCBzZWVkLCBjb3JwdXMpICU+JQogICAgICBkbyhnZXRfbG1fcmVzaWR1YWxzKC4sIGksIGZ1bGxfZW52KSkgJT4lCiAgICB1bmdyb3VwKCkKCiAgZnVsbF9yZXNpZHVhbHMgPSByYmluZChmdWxsX3Jlc2lkdWFscywgZm9sZF9mdWxsX3Jlc2lkdWFscykKfQpgYGAKCmBgYHtyfQojd3JpdGUuY3N2KGZ1bGxfcmVzaWR1YWxzLCAiLi4vZGF0YS9hbmFseXNpc19jaGVja3BvaW50cy9mdWxsX3Jlc2lkdWFscy5jc3YiKQojd3JpdGUuY3N2KGJhc2VsaW5lX3Jlc2lkdWFscywgIi4uL2RhdGEvYW5hbHlzaXNfY2hlY2twb2ludHMvYmFzZWxpbmVfcmVzaWR1YWxzLmNzdiIpCmBgYAoKYGBge3J9Cm1vZGVsX2RlbHRhcyA9IGxvZ19saWtfZGVsdGFzICU+JQogIGdyb3VwX2J5KG1vZGVsLCB0cmFpbmluZywgc2VlZCwgY29ycHVzKSAlPiUgCiAgc3VtbWFyaXNlKG1lYW5fZGVsdGFfbG9nX2xpayA9IG1lYW4oZGVsdGFfbG9nX2xpayksCiAgICAgICAgICAgIHNlbV9kZWx0YV9sb2dfbGlrID0gc2QoZGVsdGFfbG9nX2xpaykgLyBzcXJ0KGxlbmd0aChkZWx0YV9sb2dfbGlrKSkpCmBgYAoKYGBge3J9CndyaXRlLmNzdihmdWxsX21vZGVsX3Jlc3VsdHMsICIuLi9kYXRhL2FuYWx5c2lzX2NoZWNrcG9pbnRzL2Z1bGxfbW9kZWxfcmVzdWx0LmNzdiIpCndyaXRlLmNzdihiYXNlbGluZV9yZXN1bHRzLCAiLi4vZGF0YS9hbmFseXNpc19jaGVja3BvaW50cy9iYXNlbGluZV9yZXN1bHRzLmNzdiIpCiNmdWxsX21vZGVsX3Jlc3VsdHMgPSByZWFkLmNzdigiLi4vZGF0YS9hbmFseXNpc19jaGVja3BvaW50cy9mZnVsbF9tb2RlbF9yZXN1bHRzLmNzdiIpCiNiYXNlbGluZV9yZXN1bHRzID0gcmVhZC5jc3YoIi4uL2RhdGEvYW5hbHlzaXNfY2hlY2twb2ludHMvZmJhc2VsaW5lX3Jlc3VsdHNiLmNzdiIpCmBgYAoKYGBge3J9Cm1ldHJpYyA8LSAizpRMb2dMaWsiCiNtZXRyaWMgPC0gIi3OlE1TRSIKCiMgIyBTZWxlY3QgdGhlIHJlbGV2YW50IG1ldHJpYy4KbW9kZWxfZGVsdGFzID0gbW9kZWxfZGVsdGFzICU+JQogICAgIyBSZXRyaWV2ZSB0aGUgY3VycmVudCB0ZXN0IG1ldHJpYwogICAgbXV0YXRlKGRlbHRhX3Rlc3RfbWVhbiA9IG1lYW5fZGVsdGFfbG9nX2xpaywKICAgICAgICAgICBkZWx0YV90ZXN0X3NlbSA9IHNlbV9kZWx0YV9sb2dfbGlrKSAlPiUKICAgICMgbXV0YXRlKGRlbHRhX3Rlc3RfbWVhbiA9IG1lYW5fZGVsdGFfbXNlLAogICAgIyAgICAgICAgZGVsdGFfdGVzdF9zZW0gPSBzZW1fZGVsdGFfbXNlKQogICAgCiAgICAjIFJlbW92ZSB0aGUgcmF3IG1ldHJpY3MuCiAgICBzZWxlY3QoLW1lYW5fZGVsdGFfbG9nX2xpaywgLXNlbV9kZWx0YV9sb2dfbGlrLAogICAgICAgICAgICMtbWVhbl9kZWx0YV9tc2UsIC1zZW1fZGVsdGFfbXNlCiAgICAgICAgICAgKQptb2RlbF9kZWx0YXMKYGBgCgpgYGB7cn0KIyBTYW5pdHkgY2hlY2s6IHRyYWluaW5nIG9uIHRyYWluK3Rlc3QgZGF0YSBzaG91bGQgeWllbGQgaW1wcm92ZWQgcGVyZm9ybWFuY2Ugb3ZlciB0cmFpbmluZyBvbiBqdXN0IHRyYWluaW5nIGRhdGEuIChXaGVuIGV2YWx1YXRpbmcgb24gdGVzdCBkYXRhLikKIyBmdWxsX2Jhc2VsaW5lcyA9IGFsbF9kYXRhICU+JQojICAgZ3JvdXBfYnkobW9kZWwsIHRyYWluaW5nLCBzZWVkLCBjb3JwdXMpICU+JQojICAgc3VtbWFyaXNlKGJhc2VsaW5lX3RyYWluX2FsbF90ZXN0X2xpayA9IGxvZ0xpa190ZXN0KGxtKHBzeWNob21ldHJpYyB+IGxlbiArIGZyZXEgKyBzZW50X3BvcywgZGF0YT0uKSwgc2VtaV9qb2luKHRlc3RfZGF0YSwgLiwgYnk9YygidHJhaW5pbmciLCAibW9kZWwiLCAic2VlZCIsICJjb3JwdXMiKSksIHNlbWlfam9pbih0ZXN0X2RhdGEsIC4sIGJ5PWMoInRyYWluaW5nIiwgIm1vZGVsIiwgInNlZWQiLCAiY29ycHVzIikpJHBzeWNob21ldHJpYykpICU+JQojICAgdW5ncm91cCgpCiMgZnVsbF9iYXNlbGluZXMKIyAKIyBmdWxsX2Jhc2VsaW5lcyAlPiUKIyAgIHJpZ2h0X2pvaW4oYmFzZWxpbmVzLCBieT1jKCJzZWVkIiwgInRyYWluaW5nIiwgIm1vZGVsIiwgImNvcnB1cyIpKSAlPiUKIyAgIG11dGF0ZShkZWx0YT1iYXNlbGluZV90cmFpbl9hbGxfdGVzdF9saWstYmFzZWxpbmVfdGVzdF9saWspICU+JQojICAgc2VsZWN0KC1iYXNlbGluZV9saWspICMgJT4lCiMgICAjc2VsZWN0KC1iYXNlbGluZV90ZXN0X2xpaywgLWJhc2VsaW5lX3RyYWluX2FsbF90ZXN0X2xpaywgLWJhc2VsaW5lX2xpaywgLWJhc2VsaW5lX3Rlc3RfbXNlKQpgYGAKCiMgTG9hZCBsYW5ndWFnZSBtb2RlbCBkYXRhIChTeW50YXhHeW0sIFBQTCkKCmBgYHtyfQpsYW5ndWFnZV9tb2RlbF9kYXRhID0gcmVhZC5jc3YoIi4uL2RhdGEvbW9kZWxfbWV0YWRhdGEuY3N2IikgJT4lCiAgbXV0YXRlKG1vZGVsID0gYXMuY2hhcmFjdGVyKG1vZGVsKSwKICAgICAgICAgbW9kZWwgPSBpZl9lbHNlKG1vZGVsID09ICJncHQtMiIsICJncHQyIiwgbW9kZWwpLAogICAgICAgICBtb2RlbCA9IGFzLmZhY3Rvcihtb2RlbCkpICU+JQogIG11dGF0ZSh0cmFpbl9zaXplID0gY2FzZV93aGVuKHN0cl9zdGFydHModHJhaW5pbmcsICJibGxpcC1sZyIpIH4gNDIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc3RyX3N0YXJ0cyh0cmFpbmluZywgImJsbGlwLW1kIikgfiAxNSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzdHJfc3RhcnRzKHRyYWluaW5nLCAiYmxsaXAtc20iKSB+IDUsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc3RyX3N0YXJ0cyh0cmFpbmluZywgImJsbGlwLXhzIikgfiAxKSwKICAgICAgICAgCiAgICAgICAgICMgVHJhaW5pbmcgdm9jYWJ1bGFyeSB1c3VhbGx5IGNvdmFyaWVzIHdpdGggdGhlIHRyYWluaW5nIGNvcnB1cy4KICAgICAgICAgIyBCdXQgQlBFIG1vZGVscyBzaGFyZSBhIHZvY2FidWxhcnkgYWNyb3NzIHRyYWluaW5nIGNvcnBvcmEuCiAgICAgICAgIHRyYWluaW5nX3ZvY2FiPWFzLmZhY3RvcihpZmVsc2Uoc3RyX2RldGVjdCh0cmFpbmluZywgImdwdGJwZSIpLCAiZ3B0YnBlIiwgYXMuY2hhcmFjdGVyKHRyYWluaW5nKSkpLAogICAgICAgICB0cmFpbmluZ19zb3VyY2U9YXMuZmFjdG9yKHN0cl9yZXBsYWNlKGFzLmNoYXJhY3Rlcih0cmFpbmluZyksICItZ3B0YnBlIiwgIiIpKQogICAgICAgICApICU+JQogIG11dGF0ZShzZWVkID0gYXMuZmFjdG9yKHNlZWQpKSAlPiUKICBzZWxlY3QoLXBpZCwgLXRlc3RfbG9zcykgJT4lCiAgZGlzdGluY3QobW9kZWwsIHRyYWluaW5nLCBzZWVkLCAua2VlcF9hbGwgPSBUUlVFKQp0YWJsZShsYW5ndWFnZV9tb2RlbF9kYXRhJHNlZWQpCnRhYmxlKG1vZGVsX2RlbHRhcyRzZWVkKQpgYGAKCkZpcnN0IGpvaW4gZGVsdGEtbWV0cmljIGRhdGEgd2l0aCBtb2RlbCBhdXhpbGlhcnkgZGF0YS4KCmBgYHtyfQptb2RlbF9kZWx0YXMgPSBtb2RlbF9kZWx0YXMgJT4lCiAgbWVyZ2UobGFuZ3VhZ2VfbW9kZWxfZGF0YSwgYnkgPSBjKCJzZWVkIiwgInRyYWluaW5nIiwgIm1vZGVsIiksIGFsbD1UKSAlPiUKICBkcm9wX25hKCkKCm1vZGVsX2RlbHRhcwpgYGAKCkFsc28gam9pbiBvbiB0aGUgb3JpZ2luYWwgbGluZWFyIG1vZGVsIGRhdGEsIHJhdGhlciB0aGFuIGNvbGxhcHNpbmcgdG8gZGVsdGEtbWV0cmljcy4KVGhpcyB3aWxsIHN1cHBvcnQgcmVncmVzc2lvbnMgbGF0ZXIgb24gdGhhdCBkb24ndCBjb2xsYXBzZSBhY3Jvc3MgZm9sZHMuCgoKIyBGaW5hbCBkYXRhIHByZXByb2Nlc3NpbmcKCmBgYHtyIEZpbHRlciBtb2RlbHMgYW5kL29yIGNvcnBvcmF9CiMgRXhjbHVkZSBvcmRlcmVkLW5ldXJvbnMgZnJvbSBhbGwgYW5hbHlzZXMuCm1vZGVsX2RlbHRhcyA8LSBtb2RlbF9kZWx0YXMgJT4lCiAgZmlsdGVyKG1vZGVsICE9ICJvcmRlcmVkLW5ldXJvbnMiKQpgYGAKCgojIFZpc3VhbGl6YXRpb25zCgojIyBUaGUgYmFzaWNzCgpgYGB7ciwgZmlnLmNhcD0iQ29ycHVzIHNpemVzIn0KYWxsX2RhdGEgJT4lIGdncGxvdChhZXMoeD1jb3JwdXMpKSArIGdlb21fYmFyKCkKcHJpbnQoYWxsX2RhdGEgJT4lIGdyb3VwX2J5KGNvcnB1cykgJT4lIHN1bW1hcmlzZShuPW4oKSkpCmBgYAoKCmBgYHtyLCBmaWcuY2FwPSJXb3JkIGZyZXF1ZW5jeSBkaXN0cmlidXRpb24gYnkgY29ycHVzIn0KYWxsX2RhdGEgJT4lIAogIGdncGxvdChhZXMoeD1mcmVxLCBjb2xvcj1jb3JwdXMpKSArIGdlb21fZGVuc2l0eSgpCmBgYAoKYGBge3IsIGZpZy5jYXA9IldvcmQgbGVuZ3RoIGRpc3RyaWJ1dGlvbiBieSBjb3JwdXMifQphbGxfZGF0YSAlPiUgCiAgZ2dwbG90KGFlcyh4PWxlbiwgY29sb3I9Y29ycHVzKSkgKyBnZW9tX2RlbnNpdHkoKQpgYGAKCmBgYHtyLCBmaWcuY2FwPSJTdXJwcmlzYWwgZGlzdHJpYnV0aW9uIGJ5IGNvcnB1cyJ9CmFsbF9kYXRhICU+JSAKICBnZ3Bsb3QoYWVzKHg9c3VycHJpc2FsLCBjb2xvcj1jb3JwdXMpKSArIGdlb21fZGVuc2l0eSgpCmBgYAoKIyMgUHJlZGljdGl2ZSBwb3dlciBhbmQgU0cKCgpgYGB7ciBCeSBtb2RlbH0KbW9kZWxfZGVsdGFzICU+JQogIGdncGxvdChhZXMoeD1zZ19zY29yZSwgeT1kZWx0YV90ZXN0X21lYW4pKSArCiAgICBnZW9tX2Vycm9yYmFyKGFlcyh5bWluPWRlbHRhX3Rlc3RfbWVhbi1kZWx0YV90ZXN0X3NlbSwgeW1heD1kZWx0YV90ZXN0X21lYW4rZGVsdGFfdGVzdF9zZW0pKSArCiAgICBnZW9tX3Ntb290aChtZXRob2Q9ImxtIiwgc2U9VCkgKwogICAgZ2VvbV9wb2ludChzdGF0PSJpZGVudGl0eSIsIHBvc2l0aW9uPSJkb2RnZSIsIGFscGhhPTEsIHNpemU9MywgYWVzKGNvbG9yPXRyYWluaW5nX3ZvY2FiLCBzaGFwZT1tb2RlbCkpICsKICAgIHlsYWIobWV0cmljKSArCiAgICB4bGFiKCJTeW50YXggR2VuZXJhbGl6YXRpb24gU2NvcmUiKSArCiAgICBnZ3RpdGxlKCJTeW50YWN0aWMgR2VuZXJhbGl6YXRpb24gdnMuIFByZWRpY3RpdmUgUG93ZXIiKSArCiAgICBzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzID0gYygiYmxsaXAtbGciPSIjNDQwMTU0RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtbWQiPSIjMzk1NjhDRkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtc20iPSIjMUY5NjhCRkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAteHMiPSIjNzNEMDU1RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZ3B0YnBlIj0iIzg4ODg4OCIpKSArCiAgICBmYWNldF9ncmlkKH5jb3JwdXMsIHNjYWxlcz0iZnJlZSIpICsKICAgIHRoZW1lKGF4aXMudGV4dD1lbGVtZW50X3RleHQoc2l6ZT0xNCksCiAgICAgICAgICBzdHJpcC50ZXh0LnggPSBlbGVtZW50X3RleHQoc2l6ZT0xNCksCiAgICAgICAgICBsZWdlbmQudGV4dD1lbGVtZW50X3RleHQoc2l6ZT0xNCksCiAgICAgICAgICBheGlzLnRpdGxlPWVsZW1lbnRfdGV4dChzaXplPTE4KSwKICAgICAgICAgIGxlZ2VuZC5wb3NpdGlvbiA9ICJib3R0b20iKQojZ2dzYXZlKCIuL2NvZ3NjaV9pbWFnZXMvc2dfbG9nbGlrLnBuZyIsaGVpZ2h0PTUsd2lkdGg9NikKYGBgCgojIyMgUmVncmVzc2lvbiBhbmFseXNlcwoKV2UgY29udHJvbCBmb3IgZWZmZWN0cyBvZiBwZXJwbGV4aXR5IGJ5IHJlbGF0aW5nIHRoZSByZXNpZHVhbHMgb2YgYSBgcGVyZm9ybWFuY2UgfiBQUExgIHJlZ3Jlc3Npb24gdG8gU0cgc2NvcmUuCgpgYGB7ciBSZXNpZHVhbGl6ZWQgcmVncmVzc2lvbn0KIyBQcmVwYXJlIGEgcmVzaWR1YWxpemVkIHJlZ3Jlc3Npb24gZm9yIHgxIG9udG8geSwgY29udHJvbGxpbmcgZm9yIHRoZSBlZmZlY3RzIG9mIHgyLgpkX3Jlc2lkID0gbW9kZWxfZGVsdGFzICU+JQogIGRyb3BfbmEoKSAlPiUKICAKICAjIFJlc2lkdWFsaXplIGRlbHRhIG1ldHJpYyB3LnIudCBQUEwgZm9yIGVhY2ggbW9kZWwtLXRyYWluaW5nLS1zZWVkCiAgZ3JvdXBfYnkoY29ycHVzKSAlPiUKICAgIG11dGF0ZShyZXNpZC5kZWx0YSA9IHJlc2lkKGxtKGRlbHRhX3Rlc3RfbWVhbiB+IHRyYWluaW5nOnRlc3RfcHBsKSkpICU+JQogIHVuZ3JvdXAoKSAlPiUKICAKICAjIFJlc2lkdWFsaXplIFNHIHNjb3JlIHcuci50LiBQUEwgZm9yIGVhY2ggdHJhaW5pbmcgdm9jYWJ1bGFyeQogIGdyb3VwX2J5KHRyYWluaW5nX3ZvY2FiKSAlPiUKICAgICMgTkIgbm8gbmVlZCBmb3IgdHJhaW5pbmc6cHBsIGludGVyYWN0aW9uLCBzaW5jZSB3ZSdyZSB3aXRoaW4tZ3JvdXAuCiAgICBtdXRhdGUocmVzaWQuc2cgPSByZXNpZChsbShzZ19zY29yZSB+IHRlc3RfcHBsKSkpICU+JQogIHVuZ3JvdXAoKSAlPiUKICAjIENvbXB1dGUgc3VtbWFyeSBzdGF0aXN0aWNzIGFjcm9zcyBtb2RlbC0tdHJhaW5pbmctLXNlZWQtLWNvcnB1cy4KICBncm91cF9ieShtb2RlbCwgdHJhaW5pbmdfdm9jYWIsIGNvcnB1cywgc2VlZCkgJT4lCiAgICBzdW1tYXJpc2UocmVzaWQuZGVsdGEubWVhbiA9IG1lYW4ocmVzaWQuZGVsdGEpLAogICAgICAgICAgICAgIHJlc2lkLmRlbHRhLnNlbSA9IHNkKHJlc2lkLmRlbHRhKSAvIHNxcnQobGVuZ3RoKHJlc2lkLmRlbHRhKSksCiAgICAgICAgICAgICAgcmVzaWQuc2cubWVhbiA9IG1lYW4ocmVzaWQuc2cpLAogICAgICAgICAgICAgIHJlc2lkLnNnLnNlbSA9IHNkKHJlc2lkLnNnKSAvIHNxcnQobGVuZ3RoKHJlc2lkLnNnKSkpCiMgTm93IHBsb3QgcmVzaWR1YWwgdnMgU0cKZF9yZXNpZCAlPiUKICAjZmlsdGVyKGNvcnB1cyAhPSAiYm5jLWJyb3duIikgJT4lCiAgZ2dwbG90KGFlcyh4PXJlc2lkLnNnLm1lYW4sIHk9cmVzaWQuZGVsdGEubWVhbikpICsKICAgIGdlb21fZXJyb3JiYXIoYWVzKHhtaW49cmVzaWQuc2cubWVhbiAtIHJlc2lkLnNnLnNlbSwKICAgICAgICAgICAgICAgICAgICAgIHhtYXg9cmVzaWQuc2cubWVhbiArIHJlc2lkLnNnLnNlbSwKICAgICAgICAgICAgICAgICAgICAgIHltaW49cmVzaWQuZGVsdGEubWVhbiAtIHJlc2lkLmRlbHRhLnNlbSwKICAgICAgICAgICAgICAgICAgICAgIHltYXg9cmVzaWQuZGVsdGEubWVhbiArIHJlc2lkLmRlbHRhLnNlbSksIGFscGhhPTAuMykgKwogICAgZ2VvbV9zbW9vdGgobWV0aG9kPSJsbSIsIHNlPVQpICsKICAgIGdlb21fcG9pbnQoc3RhdD0iaWRlbnRpdHkiLCBwb3NpdGlvbj0iZG9kZ2UiLCBhbHBoYT0xLCBzaXplPTQsIGFlcyhzaGFwZT1tb2RlbCwgY29sb3I9dHJhaW5pbmdfdm9jYWIpKSArCiAgICB5bGFiKHBhc3RlKCJSZXNpZHVhbCIsIG1ldHJpYykpICsKICAgIHhsYWIoIlJlc2lkdWFsIFN5bnRheCBHZW5lcmFsaXphdGlvbiBTY29yZSIpICsKICAgIGdndGl0bGUoIlN5bnRhY3RpYyBHZW5lcmFsaXphdGlvbiB2cy4gUHJlZGljdGl2ZSBQb3dlciIpICsKICAgIHNjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXMgPSBjKCJibGxpcC1sZyI9IiM0NDAxNTRGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtbWQiPSIjMzk1NjhDRkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLXNtIj0iIzFGOTY4QkZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC14cyI9IiM3M0QwNTVGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZ3B0YnBlIj0iIzg4ODg4OCIpKSArCiAgICBmYWNldF9ncmlkKC5+Y29ycHVzLCBzY2FsZXM9ImZyZWUiKSArCiAgICB0aGVtZShheGlzLnRleHQ9ZWxlbWVudF90ZXh0KHNpemU9MTQpLAogICAgICAgICAgc3RyaXAudGV4dC54ID0gZWxlbWVudF90ZXh0KHNpemU9MTQpLAogICAgICAgICAgbGVnZW5kLnRleHQ9ZWxlbWVudF90ZXh0KHNpemU9MTQpLAogICAgICAgICAgYXhpcy50aXRsZT1lbGVtZW50X3RleHQoc2l6ZT0xOCksCiAgICAgICAgICBsZWdlbmQucG9zaXRpb24gPSAicmlnaHQiKQpnZ3NhdmUoIi4uL2ltYWdlcy9jdW55MjAyMC9wcGxfc2cucG5nIixoZWlnaHQ9NC41LHdpZHRoPTExKQpgYGAKCgpgYGB7ciBTdGVwd2lzZSByZWdyZXNzaW9ufQpkb19zdGVwd2lzZV9yZWdyZXNzaW9uID0gZnVuY3Rpb24oY3VyX2NvcnB1cykgewogIHJlZ3Jlc3Npb25fZGF0YSA9IG1vZGVsX2RlbHRhcyAlPiUKICAgIGZpbHRlcihjb3JwdXMgPT0gY3VyX2NvcnB1cykKICAKICBwcmludCgiLS0tLS0tLS0tLS0tLS0tLS0tLS0tLSIpCiAgcHJpbnQoY3VyX2NvcnB1cykKICAKICBsbTEgPSBsbShkZWx0YV90ZXN0X21lYW4gfiB0cmFpbmluZ192b2NhYjp0ZXN0X3BwbCwgZGF0YSA9IHJlZ3Jlc3Npb25fZGF0YSkKICBsbTIgPSBsbShkZWx0YV90ZXN0X21lYW4gfiB0cmFpbmluZ192b2NhYjp0ZXN0X3BwbCArIHNnX3Njb3JlLCBkYXRhID0gcmVncmVzc2lvbl9kYXRhKQogIHByaW50KGFub3ZhKGxtMSwgbG0yKSkKICBzdW1tYXJ5KGxtMikKfQojZG9fc3RlcHdpc2VfcmVncmVzc2lvbigiYm5jLWJyb3duIikKZG9fc3RlcHdpc2VfcmVncmVzc2lvbigiZHVuZGVlIikKZG9fc3RlcHdpc2VfcmVncmVzc2lvbigibmF0dXJhbC1zdG9yaWVzIikKYGBgCgojIyBQcmVkaWN0aXZlIHBvd2VyIGFuZCBwZXJwbGV4aXR5CgpgYGB7cn0KbW9kZWxfZGVsdGFzICU+JQogIGdncGxvdChhZXMoeD10ZXN0X3BwbCwgeT1kZWx0YV90ZXN0X21lYW4sIGNvbG9yPXRyYWluaW5nX3ZvY2FiKSkgKwogICAgZ2VvbV9lcnJvcmJhcihhZXMoeW1pbj1kZWx0YV90ZXN0X21lYW4tZGVsdGFfdGVzdF9zZW0sIHltYXg9ZGVsdGFfdGVzdF9tZWFuK2RlbHRhX3Rlc3Rfc2VtKSwgYWxwaGE9MC40KSArCiAgICAjZ2VvbV9zbW9vdGgobWV0aG9kPSJsbSIsIHNlPUYpICsKICAgIGdlb21fcG9pbnQoc3RhdD0iaWRlbnRpdHkiLCBwb3NpdGlvbj0iZG9kZ2UiLCBhbHBoYT0xLCBzaXplPTQsIGFlcyhzaGFwZT1tb2RlbCkpICsKICAgIHlsYWIobWV0cmljKSArCiAgICB4bGFiKCJUZXN0IFBlcnBsZXhpdHkiKSArCiAgICAjY29vcmRfY2FydGVzaWFuKHlsaW0gPSBjKDEsIDE2KSkgKwogICAgZ2d0aXRsZSgiVGVzdCBQZXJwbGV4aXR5IHZzLiBQcmVkaWN0aXZlIFBvd2VyIikgKwogICAgc2NhbGVfY29sb3JfbWFudWFsKHZhbHVlcyA9IGMoImJsbGlwLWxnIj0iIzQ0MDE1NEZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1tZCI9IiMzOTU2OENGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtc20iPSIjMUY5NjhCRkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLXhzIj0iIzczRDA1NUZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJncHRicGUiPSIjODg4ODg4IikpICsKICAgIGZhY2V0X2dyaWQofmNvcnB1cywgc2NhbGVzPSJmcmVlIikgKwogICAgI2Nvb3JkX2NhcnRlc2lhbih5bGltID0gYygwLCAxNTApKSArCiAgICB0aGVtZShheGlzLnRleHQ9ZWxlbWVudF90ZXh0KHNpemU9MTIpLAogICAgICAgICAgc3RyaXAudGV4dC54ID0gZWxlbWVudF90ZXh0KHNpemU9MTIpLAogICAgICAgICAgbGVnZW5kLnRleHQ9ZWxlbWVudF90ZXh0KHNpemU9MTIpLAogICAgICAgICAgYXhpcy50aXRsZT1lbGVtZW50X3RleHQoc2l6ZT0xMiksCiAgICAgICAgICBsZWdlbmQucG9zaXRpb24gPSAicmlnaHQiKQpnZ3NhdmUoIi4uL2ltYWdlcy9jdW55MjAyMC9wcGxfbG9nbGlrLnBuZyIsaGVpZ2h0PTQuNSx3aWR0aD0xMSkKCmBgYAoKCiMjIEVmZmVjdCBvZiB0cmFpbmluZyBkYXRhIHNpemUKCmBgYHtyIE9uIHByZWRpY3RpdmUgcG93ZXJ9Cm1vZGVsX2RlbHRhcyAlPiUKICBtdXRhdGUodHJhaW5fc2l6ZSA9IGxvZyh0cmFpbl9zaXplKSkgJT4lCiAgZ2dwbG90KGFlcyh4PXRyYWluX3NpemUsIHk9ZGVsdGFfdGVzdF9tZWFuLCBjb2xvcj1tb2RlbCkpICsKICAgIGdlb21fZXJyb3JiYXIoYWVzKHltaW49ZGVsdGFfdGVzdF9tZWFuLWRlbHRhX3Rlc3Rfc2VtLCB5bWF4PWRlbHRhX3Rlc3RfbWVhbitkZWx0YV90ZXN0X3NlbSksIHdpZHRoID0gMC4xKSArCiAgICBnZW9tX3Ntb290aChtZXRob2Q9ImxtIiwgc2U9VCwgYWxwaGE9MC41KSArCiAgICBnZW9tX3BvaW50KHN0YXQ9ImlkZW50aXR5IiwgcG9zaXRpb249ImRvZGdlIiwgYWxwaGE9MSwgc2l6ZT0zKSArCiAgICB5bGFiKG1ldHJpYykgKwogICAgeGxhYigiTG9nIE1pbGxpb24gVHJhaW5pbmcgVG9rZW5zIikgKwogICAgZ2d0aXRsZSgiVHJhaW5pbmcgU2l6ZSB2cy4gUHJlZGljdGl2ZSBQb3dlciIpICsKICAgIGZhY2V0X2dyaWQoY29ycHVzfm1vZGVsLCBzY2FsZXM9ImZyZWUiKSArCiAgICAjc2NhbGVfY29sb3JfbWFudWFsKHZhbHVlcyA9IGMoIiNBNDJFRjEiLCAiIzM4OTRDOCIpKSArCiAgICB0aGVtZShheGlzLnRleHQ9ZWxlbWVudF90ZXh0KHNpemU9MTQpLAogICAgICAgICAgc3RyaXAudGV4dC54ID0gZWxlbWVudF90ZXh0KHNpemU9MTQpLAogICAgICAgICAgbGVnZW5kLnRleHQ9ZWxlbWVudF90ZXh0KHNpemU9MTQpLAogICAgICAgICAgYXhpcy50aXRsZT1lbGVtZW50X3RleHQoc2l6ZT0xOCksCiAgICAgICAgICBsZWdlbmQucG9zaXRpb24gPSAiYm90dG9tIikKI2dnc2F2ZSgiLi9jb2dzY2lfaW1hZ2VzL3RyYWluaW5nX2xvZ2xpay5wbmciLGhlaWdodD01LHdpZHRoPTYpCmBgYAoKCmBgYHtyIE9uIFNHIHNjb3JlfQptb2RlbF9kZWx0YXMgJT4lCiAgbXV0YXRlKHRyYWluX3NpemUgPSBsb2codHJhaW5fc2l6ZSkpICU+JQogIGdncGxvdChhZXMoeD10cmFpbl9zaXplLCB5PXNnX3Njb3JlLCBjb2xvcj1tb2RlbCkpICsKICAgIGdlb21fc21vb3RoKG1ldGhvZD0ibG0iLCBzZT1ULCBhbHBoYT0wLjUpICsKICAgIGdlb21fcG9pbnQoc3RhdD0iaWRlbnRpdHkiLCBwb3NpdGlvbj0iZG9kZ2UiLCBhbHBoYT0xLCBzaXplPTMpICsKICAgIHlsYWIoIlNHIFNDb3JlIikgKwogICAgeGxhYigiTG9nIE1pbGxpb24gVHJhaW5pbmcgVG9rZW5zIikgKwogICAgZ2d0aXRsZSgiVHJhaW5pbmcgU2l6ZSB2cy4gU3ludGFjdGljIEdlbmVyYWxpemF0aW9uIikgKwogICAgI3NjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXMgPSBjKCIjQTQyRUYxIiwgIiMzODk0QzgiKSkgKwogICAgZmFjZXRfZ3JpZCh+bW9kZWwsIHNjYWxlcz0iZnJlZSIpICsKICAgIHRoZW1lKGF4aXMudGV4dD1lbGVtZW50X3RleHQoc2l6ZT0xNCksCiAgICAgICAgICBzdHJpcC50ZXh0LnggPSBlbGVtZW50X3RleHQoc2l6ZT0xNCksCiAgICAgICAgICBsZWdlbmQudGV4dD1lbGVtZW50X3RleHQoc2l6ZT0xNCksCiAgICAgICAgICBheGlzLnRpdGxlPWVsZW1lbnRfdGV4dChzaXplPTE4KSwKICAgICAgICAgIGxlZ2VuZC5wb3NpdGlvbiA9ICJib3R0b20iKQojZ2dzYXZlKCIuL2NvZ3NjaV9pbWFnZXMvdHJhaW5pbmdfc2cucG5nIixoZWlnaHQ9NSx3aWR0aD02KQpgYGAKCiMjIFNtaXRoICYgTGV2eSByZXByb2R1Y3Rpb24KCmBgYHtyfQphbGxfZGF0YSAlPiUKICAjZmlsdGVyKHN1cnByaXNhbCA8IDE1LCBzdXJwcmlzYWwgPiAwKSAlPiUKICBtdXRhdGUoYnBlPXN0cl9kZXRlY3QodHJhaW5pbmcsICJicGUiKSwKICAgICAgICAgdHJhaW5pbmdfc291cmNlPXN0cl9yZXBsYWNlKHRyYWluaW5nLCAiLWdwdGJwZSIsICIiKSkgJT4lIAogIGdncGxvdChhZXMoeD1zdXJwcmlzYWwsIHk9cHN5Y2hvbWV0cmljLCBjb2xvcj10cmFpbmluZ19zb3VyY2UsIGxpbmV0eXBlPWJwZSkpICsKICAgIHN0YXRfc21vb3RoKHNlPVQsIGFscGhhPTAuNSkgKwogICAgI2dlb21fZXJyb3JiYXIoY29sb3I9ImJsYWNrIiwgd2lkdGg9LjIsIHBvc2l0aW9uPXBvc2l0aW9uX2RvZGdlKHdpZHRoPS45KSwgYWxwaGE9MC4zKSArCiAgICAjZ2VvbV9wb2ludChzdGF0PSJpZGVudGl0eSIsIHBvc2l0aW9uPSJkb2RnZSIsIGFscGhhPTEsIHNpemU9MykgKwogICAgeWxhYigiUHJvY2Vzc2luZyBUaW1lIChtcykiKSArCiAgICB4bGFiKCJTdXJwcmlzYWwgKGJpdHMpIikgKwogICAgZ2d0aXRsZSgiU3VycHJpc2FsIHZzLiBSZWFkaW5nIFRpbWUgLyBHYXplIER1cmF0aW9uIikgKwogICAgZmFjZXRfZ3JpZChjb3JwdXN+bW9kZWwsIHNjYWxlcyA9ICJmcmVlIikgKwogICAgIyBzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzID0gYygiYmxsaXAtbGciPSIjNDQwMTU0RkYiLAogICAgIyAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1tZCI9IiMzOTU2OENGRiIsCiAgICAjICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLXNtIj0iIzFGOTY4QkZGIiwKICAgICMgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAteHMiPSIjNzNEMDU1RkYiLAogICAgIyAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1sZy1ncHRicGUiPSIjODg4ODg4IiwKICAgICMgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtbWQtZ3B0YnBlIj0iIzg4ODg4OCIsCiAgICAjICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLXNtLWdwdGJwZSI9IiM4ODg4ODgiLAogICAgIyAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC14cy1ncHRicGUiPSIjODg4ODg4IikpICsKICAgIHRoZW1lKGF4aXMudGV4dD1lbGVtZW50X3RleHQoc2l6ZT0xNCksCiAgICAgICAgICBheGlzLnRleHQueSA9IGVsZW1lbnRfdGV4dChzaXplID0gMTApLAogICAgICAgICAgc3RyaXAudGV4dC54ID0gZWxlbWVudF90ZXh0KHNpemU9MTQpLAogICAgICAgICAgbGVnZW5kLnRleHQ9ZWxlbWVudF90ZXh0KHNpemU9MTQpLAogICAgICAgICAgYXhpcy50aXRsZT1lbGVtZW50X3RleHQoc2l6ZT0xOCksCiAgICAgICAgICBsZWdlbmQucG9zaXRpb24gPSAicmlnaHQiKQpnZ3NhdmUoIi4uL2ltYWdlcy9jdW55MjAyMC9zdXJwX2NvcnIucG5nIixoZWlnaHQ9NC41LHdpZHRoPTEyKQpgYGAKCiMjIyBJbnZlc3RpZ2F0ZSB2YW5pbGxhCgpgYGB7cn0KYWxsX2RhdGEgJT4lCiAgI2ZpbHRlcihzdXJwcmlzYWwgPCAxNSwgc3VycHJpc2FsID4gMCkgJT4lCiAgZmlsdGVyKG1vZGVsID09ICJ2YW5pbGxhIikgJT4lIAogIGdncGxvdChhZXMoeD1zdXJwcmlzYWwsIHk9cHN5Y2hvbWV0cmljKSkgKwogICAgI3N0YXRfc21vb3RoKHNlPVQsIGFscGhhPTAuNSkgKwogICAgI2dlb21fZXJyb3JiYXIoY29sb3I9ImJsYWNrIiwgd2lkdGg9LjIsIHBvc2l0aW9uPXBvc2l0aW9uX2RvZGdlKHdpZHRoPS45KSwgYWxwaGE9MC4zKSArCiAgICBnZW9tX3BvaW50KGFscGhhPTAuMSkgKyAjc3RhdD0iaWRlbnRpdHkiLCBwb3NpdGlvbj0iZG9kZ2UiLCBhbHBoYT0xLCBzaXplPTMpICsKICAgIHlsYWIoIlByb2Nlc3NpbmcgVGltZSAobXMpIikgKwogICAgeGxhYigiU3VycHJpc2FsIChiaXRzKSIpICsKICAgIGdndGl0bGUoIlN1cnByaXNhbCB2cy4gUmVhZGluZyBUaW1lIC8gR2F6ZSBEdXJhdGlvbjogVmFuaWxsYSIpICsKICAgIGZhY2V0X2dyaWQoY29ycHVzfnRyYWluaW5nLCBzY2FsZXMgPSAiZnJlZSIpCiAgICAjIHNjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXMgPSBjKCJibGxpcC1sZyI9IiM0NDAxNTRGRiIsCiAgICAjICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLW1kIj0iIzM5NTY4Q0ZGIiwKICAgICMgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtc20iPSIjMUY5NjhCRkYiLAogICAgIyAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC14cyI9IiM3M0QwNTVGRiIsCiAgICAjICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLWxnLWdwdGJwZSI9IiM4ODg4ODgiLAogICAgIyAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1tZC1ncHRicGUiPSIjODg4ODg4IiwKICAgICMgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtc20tZ3B0YnBlIj0iIzg4ODg4OCIsCiAgICAjICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLXhzLWdwdGJwZSI9IiM4ODg4ODgiKSkKYGBgCgpgYGB7ciBUb255IEJsYWlyIGlzIHRoZSBzb3VyY2Ugb2YgdGhhdCByaWdodC1zaWRlIGRpcH0KYWxsX2RhdGEgJT4lIAogIGZpbHRlcihjb3JwdXMgPT0gImR1bmRlZSIsIG1vZGVsID09ICJ2YW5pbGxhIiwgdHJhaW5pbmcgPT0gImJsbGlwLWxnIiwgc3VycHJpc2FsID4gMjAsIHBzeWNob21ldHJpYyA8IDMwMCkKYGBgCgpgYGB7cn0KcHJpbnQoZnVsbF9yZXNpZHVhbHMgJT4lIGZpbHRlcihjb3JwdXMgPT0gImR1bmRlZSIsIG1vZGVsID09ICJ2YW5pbGxhIiwgdHJhaW5pbmcgPT0gImJsbGlwLWxnIikgJT4lIGFycmFuZ2UoZGVzYyhyZXNpZCkpKQpmdWxsX3Jlc2lkdWFscyAlPiUgZmlsdGVyKGNvcnB1cyA9PSAiZHVuZGVlIiwgbW9kZWwgPT0gInZhbmlsbGEiLCB0cmFpbmluZyA9PSAiYmxsaXAtbGciKSAlPiUgYXJyYW5nZShkZXNjKHJlc2lkKSkgJT4lIGZpbHRlcihyZXNpZCA+IDE1MCkgJT4lIAogIGdncGxvdChhZXMoeD1zdXJwcmlzYWwpKSArIGdlb21fZGVuc2l0eSgpCmBgYAoKCiMjIyBJbnZlc3RpZ2F0ZSBSTk5HCgpgYGB7cn0KYWxsX2RhdGEgJT4lCiAgI2ZpbHRlcihzdXJwcmlzYWwgPCAxNSwgc3VycHJpc2FsID4gMCkgJT4lCiAgZmlsdGVyKG1vZGVsID09ICJybm5nIikgJT4lIAogIGdncGxvdChhZXMoeD1zdXJwcmlzYWwsIHk9cHN5Y2hvbWV0cmljKSkgKwogICAgI3N0YXRfc21vb3RoKHNlPVQsIGFscGhhPTAuNSkgKwogICAgI2dlb21fZXJyb3JiYXIoY29sb3I9ImJsYWNrIiwgd2lkdGg9LjIsIHBvc2l0aW9uPXBvc2l0aW9uX2RvZGdlKHdpZHRoPS45KSwgYWxwaGE9MC4zKSArCiAgICBnZW9tX3BvaW50KGFscGhhPTAuMSkgKyAjc3RhdD0iaWRlbnRpdHkiLCBwb3NpdGlvbj0iZG9kZ2UiLCBhbHBoYT0xLCBzaXplPTMpICsKICAgIHlsYWIoIlByb2Nlc3NpbmcgVGltZSAobXMpIikgKwogICAgeGxhYigiU3VycHJpc2FsIChiaXRzKSIpICsKICAgIGdndGl0bGUoIlN1cnByaXNhbCB2cy4gUmVhZGluZyBUaW1lIC8gR2F6ZSBEdXJhdGlvbjogUk5ORyIpICsKICAgIGZhY2V0X2dyaWQoY29ycHVzfnRyYWluaW5nLCBzY2FsZXMgPSAiZnJlZSIpCmBgYAoKYGBge3IgVG9ueSBCbGFpciBpcyB0aGUgc291cmNlIG9mIHRoYXQgcmlnaHQtc2lkZSBkaXAgZm9yIFJOTkcgdG9vfQphbGxfZGF0YSAlPiUgCiAgZmlsdGVyKGNvcnB1cyA9PSAiZHVuZGVlIiwgbW9kZWwgPT0gInJubmciLCB0cmFpbmluZyA9PSAiYmxsaXAtbGciLCBzdXJwcmlzYWwgPiAyMCwgcHN5Y2hvbWV0cmljIDwgMzAwKQpgYGAKCmBgYHtyfQpwcmludChmdWxsX3Jlc2lkdWFscyAlPiUgZmlsdGVyKGNvcnB1cyA9PSAiZHVuZGVlIiwgbW9kZWwgPT0gInJubmciLCB0cmFpbmluZyA9PSAiYmxsaXAtbGciKSAlPiUgYXJyYW5nZShkZXNjKHJlc2lkKSkpCmZ1bGxfcmVzaWR1YWxzICU+JSBmaWx0ZXIoY29ycHVzID09ICJkdW5kZWUiLCBtb2RlbCA9PSAicm5uZyIsIHRyYWluaW5nID09ICJibGxpcC1sZyIpICU+JSBhcnJhbmdlKGRlc2MocmVzaWQpKSAlPiUgZmlsdGVyKHJlc2lkID4gMTUwKSAlPiUgCiAgZ2dwbG90KGFlcyh4PXN1cnByaXNhbCkpICsgZ2VvbV9kZW5zaXR5KCkKYGBgCgojIyMgSW52ZXN0aWdhdGUgbmdyYW0gdnMgdmFuaWxsYQoKYGBge3J9Cm5ncmFtX3Jlc2lkcyA9IGZ1bGxfcmVzaWR1YWxzICU+JSBmaWx0ZXIobW9kZWwgPT0gIjVncmFtIiwgdHJhaW5pbmcgPT0gImJsbGlwLXNtIikgJT4lIGdyb3VwX2J5KGNvcnB1cywgY29kZSkgJT4lIHN1bW1hcmlzZShmcmVxPW1lYW4oZnJlcSksIHBzeWNob21ldHJpYz1tZWFuKHBzeWNob21ldHJpYyksIHJlc2lkPW1lYW4ocmVzaWQpKQp2YW5pbGxhX3Jlc2lkcyA9IGZ1bGxfcmVzaWR1YWxzICU+JSBmaWx0ZXIobW9kZWwgPT0gInZhbmlsbGEiLCB0cmFpbmluZyA9PSAiYmxsaXAtc20iKSAlPiUgZ3JvdXBfYnkoY29ycHVzLCBjb2RlKSAlPiUgc3VtbWFyaXNlKGZyZXE9bWVhbihmcmVxKSwgcHN5Y2hvbWV0cmljPW1lYW4ocHN5Y2hvbWV0cmljKSwgcmVzaWQ9bWVhbihyZXNpZCkpCnJlc2lkc19qb2luZWQgPSBuZ3JhbV9yZXNpZHMgJT4lIGxlZnRfam9pbih2YW5pbGxhX3Jlc2lkcywgYnk9YygiY29ycHVzIiwgImNvZGUiKSwgc3VmZml4PWMoIi5uZ3JhbSIsICIudmFuaWxsYSIpKQoKcmVzaWRzX2pvaW5lZCAlPiUgCiAgZ2dwbG90KGFlcyh4PXJlc2lkLm5ncmFtLCB5PXJlc2lkLnZhbmlsbGEpKSArIGdlb21fcG9pbnQoKSArIGdlb21fYWJsaW5lKHNsb3BlPTEsIGNvbG9yPSJyZWQiKSArCiAgZmFjZXRfZ3JpZCh+Y29ycHVzKQoKcmVzaWRzX2pvaW5lZCAlPiUgCiAgbXV0YXRlKHJlc2lkX2Fic19kaWZmPWFicyhyZXNpZC5uZ3JhbSAtIHJlc2lkLnZhbmlsbGEpKSAlPiUgCiAgZ2dwbG90KGFlcyh4PXJlc2lkX2Fic19kaWZmKSkgKyBnZW9tX2RlbnNpdHkoKSArCiAgZmFjZXRfZ3JpZCh+Y29ycHVzKQoKcmVzaWRzX2pvaW5lZCAlPiUgCiAgbXV0YXRlKHJlc2lkX2Fic19kaWZmPWFicyhyZXNpZC5uZ3JhbSAtIHJlc2lkLnZhbmlsbGEpKSAlPiUgCiAgZ2dwbG90KGFlcyh4PWZyZXEubmdyYW0sIHk9cmVzaWRfYWJzX2RpZmYpKSArIGdlb21fcG9pbnQoYWxwaGE9MC4xKSArIGdlb21fc21vb3RoKCkKYGBgCgojIyMgSW52ZXN0aWdhdGUgZ3B0YnBlIHZzIHZhbmlsbGEKCmBgYHtyfQpncHRfcmVzaWRzID0gZnVsbF9yZXNpZHVhbHMgJT4lIGZpbHRlcihtb2RlbCA9PSAiZ3B0MiIsIHRyYWluaW5nID09ICJibGxpcC1zbS1ncHRicGUiKSAlPiUgZ3JvdXBfYnkoY29ycHVzLCBjb2RlKSAlPiUgc3VtbWFyaXNlKGZyZXE9bWVhbihmcmVxKSwgcHN5Y2hvbWV0cmljPW1lYW4ocHN5Y2hvbWV0cmljKSwgcmVzaWQ9bWVhbihyZXNpZCkpCnZhbmlsbGFfcmVzaWRzID0gZnVsbF9yZXNpZHVhbHMgJT4lIGZpbHRlcihtb2RlbCA9PSAidmFuaWxsYSIsIHRyYWluaW5nID09ICJibGxpcC1zbSIpICU+JSBncm91cF9ieShjb3JwdXMsIGNvZGUpICU+JSBzdW1tYXJpc2UoZnJlcT1tZWFuKGZyZXEpLCBwc3ljaG9tZXRyaWM9bWVhbihwc3ljaG9tZXRyaWMpLCByZXNpZD1tZWFuKHJlc2lkKSkKcmVzaWRzX2pvaW5lZCA9IGdwdF9yZXNpZHMgJT4lIGxlZnRfam9pbih2YW5pbGxhX3Jlc2lkcywgYnk9YygiY29ycHVzIiwgImNvZGUiKSwgc3VmZml4PWMoIi5ncHQiLCAiLnZhbmlsbGEiKSkKCnJlc2lkc19qb2luZWQgJT4lIAogIGdncGxvdChhZXMoeD1yZXNpZC5ncHQsIHk9cmVzaWQudmFuaWxsYSkpICsgZ2VvbV9wb2ludCgpICsgZ2VvbV9hYmxpbmUoc2xvcGU9MSwgY29sb3I9InJlZCIpICsKICBmYWNldF9ncmlkKH5jb3JwdXMpCgpyZXNpZHNfam9pbmVkICU+JSAKICBtdXRhdGUocmVzaWRfYWJzX2RpZmY9YWJzKHJlc2lkLmdwdCAtIHJlc2lkLnZhbmlsbGEpKSAlPiUgCiAgZ2dwbG90KGFlcyh4PXJlc2lkX2Fic19kaWZmKSkgKyBnZW9tX2RlbnNpdHkoKSArCiAgZmFjZXRfZ3JpZCh+Y29ycHVzKQoKcmVzaWRzX2pvaW5lZCAlPiUgCiAgbXV0YXRlKHJlc2lkX2Fic19kaWZmPWFicyhyZXNpZC5ncHQgLSByZXNpZC52YW5pbGxhKSkgJT4lIAogIGdncGxvdChhZXMoeD1mcmVxLmdwdCwgeT1yZXNpZF9hYnNfZGlmZikpICsgZ2VvbV9wb2ludChhbHBoYT0wLjEpICsgZ2VvbV9zbW9vdGgoKQpgYGAKCiMjIyBJbnZlc3RpZ2F0ZSByZXNpZHVhbHMgb3ZlcmFsbAoKYGBge3J9CmZ1bGxfcmVzaWR1YWxzICU+JSByaWdodF9qb2luKGJhc2VsaW5lX3Jlc2lkdWFscywgYnk9YygiY29ycHVzIiwgImNvZGUiLCAibW9kZWwiLCAidHJhaW5pbmciLCAic2VlZCIpLCBzdWZmaXg9YygiLmZ1bGwiLCAiLmJhc2VsaW5lIikpICU+JSAKICBtdXRhdGUocmVzaWRfZGVsdGE9cmVzaWQuZnVsbCAtIHJlc2lkLmJhc2VsaW5lLAogICAgICAgICB0cmFpbmluZ19zb3VyY2U9YXMuZmFjdG9yKHN0cl9yZXBsYWNlKHRyYWluaW5nLCAiLWdwdGJwZSIsICIiKSksCiAgICAgICAgIGJwZT1zdHJfZGV0ZWN0KHRyYWluaW5nLCAiZ3B0YnBlIikpICU+JSAKICBnZ3Bsb3QoYWVzKHg9c3VycHJpc2FsLmZ1bGwsIHk9cmVzaWRfZGVsdGEsIGNvbG9yPXRyYWluaW5nKSkgKwogICAgZmFjZXRfZ3JpZChtb2RlbH5jb3JwdXMpICsKICAgIGdlb21fcG9pbnQoYWxwaGE9MC4xLCBzaXplPTAuNSkKYGBgCgpgYGB7cn0KbGFuZ3VhZ2VfbW9kZWxfZGF0YSAlPiUgZmlsdGVyKG1vZGVsID09ICJncHQyIikKYGBgCgo=